Skip to content

Commit 57f6c3f

Browse files
olavloiteolavloite
authored
test: add benchmarking wrapper (#749)
* feat: support env variable SPANNER_EMULATOR_HOST The Spanner client will now connect to a custom host if the environment variable SPANNER_EMULATOR_HOST has been set to a valid host:port combination. * test: add benchwrapper * fix: rename to benchwrapper_test_client.js
1 parent 883b4fa commit 57f6c3f

8 files changed

Lines changed: 420 additions & 1 deletion

File tree

‎handwritten/spanner/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@ package-lock.json
1313
.vscode
1414
yarn.lock
1515
__pycache__
16+
.idea

‎handwritten/spanner/bin/README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# benchwrapper
2+
3+
benchwrapper is a lightweight gRPC server that wraps the Spanner library for
4+
benchmarking purposes.
5+
6+
## Running
7+
8+
```
9+
cd nodejs-spanner
10+
npm install
11+
export SPANNER_EMULATOR_HOST=localhost:8080
12+
npm run benchwrapper -- --port 8081
13+
```

‎handwritten/spanner/bin/benchwrapper.js‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
const grpc = require('grpc');
16+
const protoLoader = require('@grpc/proto-loader');
17+
const {Spanner} = require('../build/src');
18+
19+
const argv = require('yargs')
20+
.option('port', {
21+
description: 'The port that the Node.js benchwrapper should run on.',
22+
type: 'number',
23+
demand: true,
24+
})
25+
.parse();
26+
27+
const PROTO_PATH = __dirname + '/spanner.proto';
28+
const packageDefinition = protoLoader.loadSync(PROTO_PATH, {
29+
keepCase: true,
30+
longs: String,
31+
enums: String,
32+
defaults: true,
33+
oneofs: true,
34+
});
35+
const protoDescriptor = grpc.loadPackageDefinition(packageDefinition);
36+
const spannerBenchWrapper = protoDescriptor.spanner_bench;
37+
38+
// The benchwrapper should only be executed against an emulator.
39+
if (!process.env.SPANNER_EMULATOR_HOST) {
40+
throw new Error(
41+
'This benchmarking server only works when connected to an emulator. Please set SPANNER_EMULATOR_HOST.'
42+
);
43+
}
44+
// This will connect the Spanner client to an emulator, as SPANNER_EMULATOR_HOST has been set.
45+
const spannerClient = new Spanner();
46+
47+
// Implementation of SpannerBenchWrapper.Read method.
48+
function Read(call, callback) {
49+
const instance = spannerClient.instance('someinstance');
50+
const database = instance.database('somedatabase');
51+
let tx;
52+
database
53+
.getSnapshot()
54+
.then(data => {
55+
tx = data[0];
56+
return tx.run(call.request.Query);
57+
})
58+
.then(data => {
59+
const [rows] = data;
60+
// Just iterate over all rows.
61+
rows.forEach(() => {});
62+
})
63+
.finally(() => {
64+
if (tx) {
65+
tx.end();
66+
}
67+
callback(null, {});
68+
});
69+
}
70+
71+
// Implementation of SpannerBenchWrapper.Insert method.
72+
function Insert(call, callback) {
73+
const instance = spannerClient.instance('someinstance');
74+
const database = instance.database('somedatabase');
75+
database.runTransaction(function(err, transaction) {
76+
if (err) {
77+
callback(err);
78+
return;
79+
}
80+
call.request.users.forEach(user => {
81+
transaction.insert('sometable', {
82+
name: user.name,
83+
age: user.age,
84+
});
85+
});
86+
transaction.commit(function(err) {
87+
if (err) {
88+
callback(err);
89+
} else {
90+
callback(null, {});
91+
}
92+
});
93+
});
94+
}
95+
96+
// Implementation of SpannerBenchWrapper.Insert method.
97+
function Update(call, callback) {
98+
const instance = spannerClient.instance('someinstance');
99+
const database = instance.database('somedatabase');
100+
database.runTransaction((err, transaction) => {
101+
if (err) {
102+
callback(err);
103+
return;
104+
}
105+
transaction.batchUpdate(call.request.Queries, (err, rowCounts) => {
106+
if (err) {
107+
callback(
108+
new grpc.StatusBuilder()
109+
.withCode(err.code)
110+
.withDetails(err.details || err.message)
111+
.withMetadata(err.metadata)
112+
.build()
113+
);
114+
transaction.rollback().then(() => {});
115+
return;
116+
}
117+
// Iterate over all rowCounts.
118+
rowCounts.forEach(() => {});
119+
transaction.commit(function(err) {
120+
if (err) {
121+
callback(err);
122+
} else {
123+
callback(null, {});
124+
}
125+
});
126+
});
127+
});
128+
}
129+
130+
// Create and start a benchwrapper server.
131+
const server = new grpc.Server();
132+
server.addService(spannerBenchWrapper['SpannerBenchWrapper']['service'], {
133+
Read: Read,
134+
Insert: Insert,
135+
Update: Update,
136+
});
137+
console.log('starting benchwrapper for Spanner on localhost:' + argv.port);
138+
server.bind('0.0.0.0:' + argv.port, grpc.ServerCredentials.createInsecure());
139+
server.start();

‎handwritten/spanner/bin/benchwrapper_test_client.js‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
// This is a simple test client for the Spanner benchwrapper.
16+
17+
const grpc = require('grpc');
18+
const protoLoader = require('@grpc/proto-loader');
19+
20+
const argv = require('yargs')
21+
.option('port', {
22+
description: 'The port that the benchwrapper client should connect to.',
23+
type: 'number',
24+
demand: true,
25+
})
26+
.parse();
27+
28+
const PROTO_PATH = __dirname + '/spanner.proto';
29+
const packageDefinition = protoLoader.loadSync(PROTO_PATH, {
30+
keepCase: true,
31+
longs: String,
32+
enums: String,
33+
defaults: true,
34+
oneofs: true,
35+
});
36+
const protoDescriptor = grpc.loadPackageDefinition(packageDefinition);
37+
const spannerBenchWrapper = protoDescriptor.spanner_bench;
38+
39+
console.log(`connecting to localhost:${argv.port}`);
40+
const client = new spannerBenchWrapper.SpannerBenchWrapper(
41+
`localhost:${argv.port}`,
42+
grpc.credentials.createInsecure()
43+
);
44+
const readReq = {
45+
Query: 'SELECT 1 AS COL1 UNION ALL SELECT 2 AS COL1',
46+
};
47+
const insertReq = {
48+
users: [
49+
{name: 'foo', age: '50'},
50+
{name: 'bar', age: '40'},
51+
],
52+
};
53+
const updateReq = {
54+
Queries: [
55+
'UPDATE sometable SET foo=1 WHERE bar=2',
56+
'UPDATE sometable SET foo=2 WHERE bar=1',
57+
],
58+
};
59+
client.read(readReq, (err, result) => {
60+
callback('read', err, result);
61+
});
62+
client.insert(insertReq, (err, result) => {
63+
callback('insert', err, result);
64+
});
65+
client.update(updateReq, (err, result) => {
66+
callback('update', err, result);
67+
});
68+
69+
function callback(method, err, result) {
70+
if (err) {
71+
console.log(`${method} failed with error ${err}`);
72+
return;
73+
}
74+
console.log(`${method} executed with result ${result}`);
75+
}

‎handwritten/spanner/bin/spanner.proto‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright 2019 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
syntax = "proto3";
16+
17+
package spanner_bench;
18+
option java_multiple_files = true;
19+
option java_package = "com.google.cloud.benchwrapper";
20+
21+
message ReadQuery{
22+
// The query to use in the read call.
23+
string Query = 1;
24+
}
25+
26+
message User{
27+
string name = 1;
28+
int64 age = 2;
29+
}
30+
31+
message InsertQuery{
32+
// The query to use in the insert call.
33+
repeated User users = 1;
34+
}
35+
36+
message UpdateQuery{
37+
// The queries to use in the update call.
38+
repeated string Queries = 1;
39+
}
40+
41+
message EmptyResponse{
42+
}
43+
44+
service SpannerBenchWrapper{
45+
// Read represents operations like Go's ReadOnlyTransaction.Query, Java's
46+
// ReadOnlyTransaction.executeQuery, Python's snapshot.read, and Node's
47+
// Transaction.Read.
48+
//
49+
// It will typically be used to read many items.
50+
rpc Read(ReadQuery) returns (EmptyResponse){}
51+
52+
// Insert represents operations like Go's Client.Apply, Java's
53+
// DatabaseClient.writeAtLeastOnce, Python's transaction.commit, and Node's
54+
// Transaction.Commit.
55+
//
56+
// It will typically be used to insert many items.
57+
rpc Insert(InsertQuery) returns (EmptyResponse){}
58+
59+
// Update represents operations like Go's ReadWriteTransaction.BatchUpdate,
60+
// Java's TransactionRunner.run, Python's Batch.update, and Node's
61+
// Transaction.BatchUpdate.
62+
//
63+
// It will typically be used to update many items.
64+
rpc Update(UpdateQuery) returns (EmptyResponse){}
65+
}

‎handwritten/spanner/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@
4747
"proto:spanner_instance_admin": "pbjs -t static-module -w commonjs -p node_modules/google-proto-files google/spanner/admin/instance/v1/spanner_instance_admin.proto | pbts -o proto/spanner_instance_admin.d.ts -",
4848
"proto:spanner_database_admin": "pbjs -t static-module -w commonjs -p node_modules/google-proto-files google/spanner/admin/database/v1/spanner_database_admin.proto | pbts -o proto/spanner_database_admin.d.ts -",
4949
"docs-test": "linkinator docs",
50-
"predocs-test": "npm run docs"
50+
"predocs-test": "npm run docs",
51+
"benchwrapper": "node bin/benchwrapper.js"
5152
},
5253
"dependencies": {
5354
"@google-cloud/common": "^2.2.2",
@@ -74,6 +75,7 @@
7475
"through2": "^3.0.0"
7576
},
7677
"devDependencies": {
78+
"@grpc/proto-loader": "^0.5.1",
7779
"@types/concat-stream": "^1.6.0",
7880
"@types/extend": "^3.0.0",
7981
"@types/is": "0.0.21",

‎handwritten/spanner/src/index.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,39 @@ class Spanner extends GrpcService {
212212
*/
213213
static COMMIT_TIMESTAMP = 'spanner.commit_timestamp()';
214214

215+
/**
216+
* Gets the configured Spanner emulator host from an environment variable.
217+
*/
218+
static getSpannerEmulatorHost():
219+
| {endpoint: string; port?: number}
220+
| undefined {
221+
const endpointWithPort = process.env.SPANNER_EMULATOR_HOST;
222+
if (endpointWithPort) {
223+
if (
224+
endpointWithPort.startsWith('http:') ||
225+
endpointWithPort.startsWith('https:')
226+
) {
227+
throw new Error(
228+
'SPANNER_EMULATOR_HOST must not start with a protocol specification (http/https)'
229+
);
230+
}
231+
const index = endpointWithPort.indexOf(':');
232+
if (index > -1) {
233+
const portName = endpointWithPort.substring(index + 1);
234+
const port = +portName;
235+
if (!port || port < 1 || port > 65535) {
236+
throw new Error(`Invalid port number: ${portName}`);
237+
}
238+
return {
239+
endpoint: endpointWithPort.substring(0, index),
240+
port: +endpointWithPort.substring(index + 1),
241+
};
242+
}
243+
return {endpoint: endpointWithPort};
244+
}
245+
return undefined;
246+
}
247+
215248
constructor(options?: SpannerOptions) {
216249
const scopes: Array<{}> = [];
217250
const clientClasses = [
@@ -239,6 +272,16 @@ class Spanner extends GrpcService {
239272
},
240273
options || {}
241274
) as {}) as SpannerOptions;
275+
const emulatorHost = Spanner.getSpannerEmulatorHost();
276+
if (
277+
emulatorHost &&
278+
emulatorHost.endpoint &&
279+
emulatorHost.endpoint.length > 0
280+
) {
281+
options.servicePath = emulatorHost.endpoint;
282+
options.port = emulatorHost.port;
283+
options.sslCreds = grpc.credentials.createInsecure();
284+
}
242285
const config = ({
243286
baseUrl:
244287
options.apiEndpoint ||

0 commit comments

Comments
 (0)