From 9445a81b5011ca66738c4803b734fb8ea435f3b8 Mon Sep 17 00:00:00 2001 From: Denis Badurina Date: Thu, 7 Oct 2021 21:19:38 +0200 Subject: [PATCH] test: Benchmark (#239) --- .github/workflows/benchmark.yml | 122 ++++++++++++++++ benchmark/k6.mjs | 148 ++++++++++++++++++++ benchmark/servers/fastify-websocket_ws8.mjs | 20 +++ benchmark/servers/index.mjs | 5 + benchmark/servers/legacy_ws7.mjs | 53 +++++++ benchmark/servers/ports.mjs | 7 + benchmark/servers/schema.mjs | 27 ++++ benchmark/servers/uWebSockets.mjs | 13 ++ benchmark/servers/ws7.mjs | 13 ++ benchmark/servers/ws8.mjs | 13 ++ package.json | 6 +- yarn.lock | 47 ++++++- 12 files changed, 472 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 benchmark/k6.mjs create mode 100644 benchmark/servers/fastify-websocket_ws8.mjs create mode 100644 benchmark/servers/index.mjs create mode 100644 benchmark/servers/legacy_ws7.mjs create mode 100644 benchmark/servers/ports.mjs create mode 100644 benchmark/servers/schema.mjs create mode 100644 benchmark/servers/uWebSockets.mjs create mode 100644 benchmark/servers/ws7.mjs create mode 100644 benchmark/servers/ws8.mjs diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..e07971db --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,122 @@ +name: Benchmark + +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened] + branches: + - master + +jobs: + uWebSockets: + name: uWebSockets + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up node + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'yarn' + - name: Install + run: yarn install --immutable + - name: Download k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + - name: Build + run: yarn run build:esm + - name: Run + run: | + NODE_ENV=production node benchmark/servers/uWebSockets.mjs & + SERVER=uWebSockets ./k6 run benchmark/k6.mjs + ws7: + name: ws7 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up node + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'yarn' + - name: Install + run: yarn install --immutable + - name: Download k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + - name: Build + run: yarn run build:esm + - name: Run + run: | + NODE_ENV=production node benchmark/servers/ws7.mjs & + SERVER=ws7 ./k6 run benchmark/k6.mjs + ws8: + name: ws8 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up node + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'yarn' + - name: Install + run: yarn install --immutable + - name: Download k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + - name: Build + run: yarn run build:esm + - name: Run + run: | + NODE_ENV=production node benchmark/servers/ws8.mjs & + SERVER=ws8 ./k6 run benchmark/k6.mjs + fastify-websocket_ws8: + name: fastify-websocket_ws8 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up node + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'yarn' + - name: Install + run: yarn install --immutable + - name: Download k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + - name: Build + run: yarn run build:esm + - name: Run + run: | + NODE_ENV=production node benchmark/servers/fastify-websocket_ws8.mjs & + SERVER=fastify-websocket_ws8 ./k6 run benchmark/k6.mjs + legacy_ws7: + name: legacy_ws7 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up node + uses: actions/setup-node@v2 + with: + node-version: '16' + cache: 'yarn' + - name: Install + run: yarn install --immutable + - name: Download k6 + run: | + curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1 + - name: Build + run: yarn run build:esm + - name: Run + run: | + NODE_ENV=production node benchmark/servers/legacy_ws7.mjs & + LEGACY=1 SERVER=legacy_ws7 ./k6 run benchmark/k6.mjs diff --git a/benchmark/k6.mjs b/benchmark/k6.mjs new file mode 100644 index 00000000..02f504ac --- /dev/null +++ b/benchmark/k6.mjs @@ -0,0 +1,148 @@ +import { check, fail } from 'k6'; +import ws from 'k6/ws'; +import { Counter, Trend } from 'k6/metrics'; +import { uuidv4 } from 'https://jslib.k6.io/k6-utils/1.0.0/index.js'; +import { ports } from './servers/ports.mjs'; +import { MessageType } from '../lib/common.mjs'; + +if (!__ENV.SERVER) { + throw new Error('SERVER not specified.'); +} + +export const options = { + scenarios: { + query: { + executor: 'constant-vus', + exec: 'run', + vus: 20, + }, + subscription: { + executor: 'constant-vus', + exec: 'run', + vus: 20, + + env: { SUBSCRIPTION: '1' }, + }, + }, +}; + +const duration = 10, // seconds + gracefulStop = 5; // seconds +let i = 0; +for (const [, scenario] of Object.entries(options.scenarios)) { + i++; + + scenario.duration = duration + 's'; + scenario.gracefulStop = gracefulStop + 's'; + if (i > 1) { + scenario.startTime = (duration + gracefulStop) * (i - 1) + 's'; + } +} + +const scenarioMetrics = {}; +for (let scenario in options.scenarios) { + if (!options.scenarios[scenario].env) options.scenarios[scenario].env = {}; + options.scenarios[scenario].env['SCENARIO'] = scenario; + if (!options.scenarios[scenario].tags) options.scenarios[scenario].tags = {}; + options.scenarios[scenario].tags['SCENARIO'] = scenario; + + scenarioMetrics[scenario] = { + runs: new Counter(`${scenario} - runs`), + opened: new Trend(`${scenario} - opened`, true), + subscribed: new Trend(`${scenario} - subscribed`, true), + completions: new Counter(`${scenario} - completions`), + completed: new Trend(`${scenario} - completed`, true), + }; +} + +export function run() { + const start = Date.now(); + + const metrics = scenarioMetrics[__ENV.SCENARIO]; + metrics.runs.add(1); + + let completed = false; + try { + ws.connect( + `ws://localhost:${ports[__ENV.SERVER]}/graphql`, + { + headers: { + 'Sec-WebSocket-Protocol': __ENV.LEGACY + ? 'graphql-ws' + : 'graphql-transport-ws', + }, + }, + function (socket) { + // each run's socket can be open for no more than 3 seconds + socket.setTimeout(() => socket.close(), 3000); + + socket.on('close', (code) => { + if (code !== 1000) throw null; + }); + + socket.on('open', () => { + metrics.opened.add(Date.now() - start); + + socket.send( + JSON.stringify({ + type: __ENV.LEGACY + ? 'connection_init' + : MessageType.ConnectionInit, + }), + ); + }); + + let msgs = 0; + socket.on('message', (data) => { + msgs++; + + if (msgs === 1) { + assertMessageType( + JSON.parse(data).type, + __ENV.LEGACY ? 'connection_ack' : MessageType.ConnectionAck, + ); + + socket.send( + JSON.stringify({ + type: __ENV.LEGACY ? 'start' : MessageType.Subscribe, + id: uuidv4(), + payload: { + query: __ENV.SUBSCRIPTION + ? 'subscription { greetings }' + : '{ hello }', + }, + }), + ); + + metrics.subscribed.add(Date.now() - start); + } else if (__ENV.SUBSCRIPTION ? msgs > 1 && msgs <= 6 : msgs === 2) { + assertMessageType( + JSON.parse(data).type, + __ENV.LEGACY ? 'data' : MessageType.Next, + ); + } else if (__ENV.SUBSCRIPTION ? msgs > 6 : msgs === 3) { + assertMessageType( + JSON.parse(data).type, + __ENV.LEGACY ? 'complete' : MessageType.Complete, + ); + + // we're done once completed + socket.close(1000); + } else fail(`Shouldn't have msgs ${msgs} messages`); + }); + }, + ); + completed = true; + metrics.completed.add(Date.now() - start); + metrics.completions.add(1); + } catch (_err) { + // noop + } + check(0, { [`${__ENV.SCENARIO} - completed`]: () => completed }); +} + +function assertMessageType(got, expected) { + if (got !== expected) { + fail(`Expected ${expected} message, got ${got}`); + } +} diff --git a/benchmark/servers/fastify-websocket_ws8.mjs b/benchmark/servers/fastify-websocket_ws8.mjs new file mode 100644 index 00000000..c84a5b9d --- /dev/null +++ b/benchmark/servers/fastify-websocket_ws8.mjs @@ -0,0 +1,20 @@ +import Fastify from 'fastify'; +import fastifyWebsocket from 'fastify-websocket'; +import { ports } from './ports.mjs'; +import { makeHandler } from '../../lib/use/fastify-websocket.mjs'; +import { schema } from './schema.mjs'; + +const fastify = Fastify(); +fastify.register(fastifyWebsocket); + +fastify.get('/graphql', { websocket: true }, makeHandler({ schema })); + +fastify.listen(ports['fastify-websocket_ws8'], (err) => { + if (err) { + fastify.log.error(err); + return process.exit(1); + } + console.log( + `fastify-websocket_ws8 - listening on port ${ports['fastify-websocket_ws8']}...`, + ); +}); diff --git a/benchmark/servers/index.mjs b/benchmark/servers/index.mjs new file mode 100644 index 00000000..d61ea00b --- /dev/null +++ b/benchmark/servers/index.mjs @@ -0,0 +1,5 @@ +import './uWebSockets.mjs'; +import './ws8.mjs'; +import './ws7.mjs'; +import './fastify-websocket_ws8.mjs'; +import './legacy_ws7.mjs'; diff --git a/benchmark/servers/legacy_ws7.mjs b/benchmark/servers/legacy_ws7.mjs new file mode 100644 index 00000000..3959e7bf --- /dev/null +++ b/benchmark/servers/legacy_ws7.mjs @@ -0,0 +1,53 @@ +import { createServer } from 'http'; +import { ports } from './ports.mjs'; +import { SubscriptionServer } from 'subscriptions-transport-ws'; +import { execute, subscribe } from 'graphql'; +import { schema } from './schema.mjs'; + +const server = createServer((_req, res) => { + res.writeHead(404); + res.end(); +}); + +SubscriptionServer.create( + { + schema, + execute: ( + schema, + document, + rootValue, + contextValue, + variableValues, + operationName, + ) => + execute({ + schema, + document, + rootValue, + contextValue, + variableValues, + operationName, + }), + subscribe: ( + schema, + document, + rootValue, + contextValue, + variableValues, + operationName, + ) => + subscribe({ + schema, + document, + rootValue, + contextValue, + variableValues, + operationName, + }), + }, + { server, path: '/graphql' }, +); + +server.listen(ports.legacy_ws7, () => { + console.log(`legacy_ws7 - listening on port ${ports.legacy_ws7}...`); +}); diff --git a/benchmark/servers/ports.mjs b/benchmark/servers/ports.mjs new file mode 100644 index 00000000..2ba151c9 --- /dev/null +++ b/benchmark/servers/ports.mjs @@ -0,0 +1,7 @@ +export const ports = { + ws8: 6540, + ws7: 6543, + uWebSockets: 6541, + legacy_ws7: 6542, + 'fastify-websocket_ws8': 6544, +}; diff --git a/benchmark/servers/schema.mjs b/benchmark/servers/schema.mjs new file mode 100644 index 00000000..e986fc57 --- /dev/null +++ b/benchmark/servers/schema.mjs @@ -0,0 +1,27 @@ +import { GraphQLSchema, GraphQLObjectType, GraphQLString } from 'graphql'; + +export const schema = new GraphQLSchema({ + query: new GraphQLObjectType({ + name: 'Query', + fields: { + hello: { + type: GraphQLString, + resolve: () => 'world', + }, + }, + }), + subscription: new GraphQLObjectType({ + name: 'Subscription', + fields: { + greetings: { + type: GraphQLString, + subscribe: async function* () { + for (const hi of ['Hi', 'Bonjour', 'Hola', 'Ciao', 'Zdravo']) { + yield { greetings: hi }; + await new Promise((resolve) => setImmediate(resolve)); + } + }, + }, + }, + }), +}); diff --git a/benchmark/servers/uWebSockets.mjs b/benchmark/servers/uWebSockets.mjs new file mode 100644 index 00000000..27c9f099 --- /dev/null +++ b/benchmark/servers/uWebSockets.mjs @@ -0,0 +1,13 @@ +import uWS from 'uWebSockets.js'; +import { ports } from './ports.mjs'; +import { makeBehavior } from '../../lib/use/uWebSockets.mjs'; +import { schema } from './schema.mjs'; + +uWS + .App() + .ws('/graphql', makeBehavior({ schema })) + .listen(ports.uWebSockets, (listenSocket) => { + if (listenSocket) { + console.log(`uWebSockets - listening on port ${ports.uWebSockets}...`); + } + }); diff --git a/benchmark/servers/ws7.mjs b/benchmark/servers/ws7.mjs new file mode 100644 index 00000000..cdf75052 --- /dev/null +++ b/benchmark/servers/ws7.mjs @@ -0,0 +1,13 @@ +import ws from 'ws7'; +import { ports } from './ports.mjs'; +import { useServer } from '../../lib/use/ws.mjs'; +import { schema } from './schema.mjs'; + +const server = new ws.Server({ + port: ports.ws7, + path: '/graphql', +}); + +useServer({ schema }, server); + +console.log(`ws7 - listening on port ${ports.ws7}...`); diff --git a/benchmark/servers/ws8.mjs b/benchmark/servers/ws8.mjs new file mode 100644 index 00000000..f6a306d4 --- /dev/null +++ b/benchmark/servers/ws8.mjs @@ -0,0 +1,13 @@ +import { WebSocketServer } from 'ws'; +import { ports } from './ports.mjs'; +import { useServer } from '../../lib/use/ws.mjs'; +import { schema } from './schema.mjs'; + +const server = new WebSocketServer({ + port: ports.ws8, + path: '/graphql', +}); + +useServer({ schema }, server); + +console.log(`ws8 - listening on port ${ports.ws8}...`); diff --git a/package.json b/package.json index 624b5946..cb29acfb 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,8 @@ "lint": "eslint 'src'", "type-check": "tsc --noEmit", "test": "jest", + "bench:start-servers": "NODE_ENV=production node benchmark/servers/index.mjs", + "bench": "k6 run benchmark/k6.mjs", "build:esm": "tsc -b tsconfig.esm.json && node scripts/esm-post-process.js", "build:cjs": "tsc -b tsconfig.cjs.json", "build:umd": "rollup -c && gzip umd/graphql-ws.min.js -c > umd/graphql-ws.min.js.gz", @@ -107,11 +109,13 @@ "rollup": "^2.56.3", "rollup-plugin-terser": "^7.0.2", "semantic-release": "^17.4.7", + "subscriptions-transport-ws": "^0.9.19", "tslib": "^2.3.1", "typedoc": "0.21.9", "typedoc-plugin-markdown": "^3.10.4", "typescript": "^4.4.3", "uWebSockets.js": "uNetworking/uWebSockets.js#v19.3.0", - "ws": "^8.2.2" + "ws": "^8.2.2", + "ws7": "npm:ws@7" } } diff --git a/yarn.lock b/yarn.lock index 60388fc1..3529708f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2860,6 +2860,13 @@ __metadata: languageName: node linkType: hard +"backo2@npm:^1.0.2": + version: 1.0.2 + resolution: "backo2@npm:1.0.2" + checksum: fda8d0a0f4810068d23715f2f45153146d6ee8f62dd827ce1e0b6cc3c8328e84ad61e11399a83931705cef702fe7cbb457856bf99b9bd10c4ed57b0786252385 + languageName: node + linkType: hard + "balanced-match@npm:^1.0.0": version: 1.0.2 resolution: "balanced-match@npm:1.0.2" @@ -4052,6 +4059,13 @@ __metadata: languageName: node linkType: hard +"eventemitter3@npm:^3.1.0": + version: 3.1.2 + resolution: "eventemitter3@npm:3.1.2" + checksum: 81e4e82b8418f5cfd986d2b4a2fa5397ac4eb8134e09bcb47005545e22fdf8e9e61d5c053d34651112245aae411bdfe6d0ad5511da0400743fef5fc38bfcfbe3 + languageName: node + linkType: hard + "execa@npm:^4.0.0": version: 4.1.0 resolution: "execa@npm:4.1.0" @@ -4695,12 +4709,14 @@ __metadata: rollup: ^2.56.3 rollup-plugin-terser: ^7.0.2 semantic-release: ^17.4.7 + subscriptions-transport-ws: ^0.9.19 tslib: ^2.3.1 typedoc: 0.21.9 typedoc-plugin-markdown: ^3.10.4 typescript: ^4.4.3 uWebSockets.js: "uNetworking/uWebSockets.js#v19.3.0" ws: ^8.2.2 + ws7: "npm:ws@7" peerDependencies: graphql: ">=0.11 <=16" languageName: unknown @@ -5309,6 +5325,13 @@ __metadata: languageName: node linkType: hard +"iterall@npm:^1.2.1": + version: 1.3.0 + resolution: "iterall@npm:1.3.0" + checksum: c78b99678f8c99be488cca7f33e4acca9b72c1326e050afbaf023f086e55619ee466af0464af94a0cb3f292e60cb5bac53a8fd86bd4249ecad26e09f17bb158b + languageName: node + linkType: hard + "java-properties@npm:^1.0.0": version: 1.0.2 resolution: "java-properties@npm:1.0.2" @@ -8698,6 +8721,21 @@ __metadata: languageName: node linkType: hard +"subscriptions-transport-ws@npm:^0.9.19": + version: 0.9.19 + resolution: "subscriptions-transport-ws@npm:0.9.19" + dependencies: + backo2: ^1.0.2 + eventemitter3: ^3.1.0 + iterall: ^1.2.1 + symbol-observable: ^1.0.4 + ws: ^5.2.0 || ^6.0.0 || ^7.0.0 + peerDependencies: + graphql: ">=0.10.0" + checksum: 6979b36e03c0a48e33836cb412941e41bae8743767dff2aa453159cfffa983b879cc80cd4e744e82afbf11062c66899c37f5dca0281253ee240098ada0078533 + languageName: node + linkType: hard + "supports-color@npm:^5.3.0": version: 5.5.0 resolution: "supports-color@npm:5.5.0" @@ -8735,6 +8773,13 @@ __metadata: languageName: node linkType: hard +"symbol-observable@npm:^1.0.4": + version: 1.2.0 + resolution: "symbol-observable@npm:1.2.0" + checksum: 48ffbc22e3d75f9853b3ff2ae94a44d84f386415110aea5effc24d84c502e03a4a6b7a8f75ebaf7b585780bda34eb5d6da3121f826a6f93398429d30032971b6 + languageName: node + linkType: hard + "symbol-tree@npm:^3.2.4": version: 3.2.4 resolution: "symbol-tree@npm:3.2.4" @@ -9464,7 +9509,7 @@ typescript@^4.4.3: languageName: node linkType: hard -"ws@npm:^7.4.6": +"ws7@npm:ws@7, ws@npm:^5.2.0 || ^6.0.0 || ^7.0.0, ws@npm:^7.4.6": version: 7.5.5 resolution: "ws@npm:7.5.5" peerDependencies: