test: Benchmark (#239)

.github/workflows/benchmark.yml

@@ -0,0 +1,122 @@
+name: Benchmark
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - master
+
+jobs:
+  uWebSockets:
+    name: uWebSockets
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+      - name: Install
+        run: yarn install --immutable
+      - name: Download k6
+        run: |
+          curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+      - name: Build
+        run: yarn run build:esm
+      - name: Run
+        run: |
+          NODE_ENV=production node benchmark/servers/uWebSockets.mjs &
+          SERVER=uWebSockets ./k6 run benchmark/k6.mjs
+  ws7:
+    name: ws7
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+      - name: Install
+        run: yarn install --immutable
+      - name: Download k6
+        run: |
+          curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+      - name: Build
+        run: yarn run build:esm
+      - name: Run
+        run: |
+          NODE_ENV=production node benchmark/servers/ws7.mjs &
+          SERVER=ws7 ./k6 run benchmark/k6.mjs
+  ws8:
+    name: ws8
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+      - name: Install
+        run: yarn install --immutable
+      - name: Download k6
+        run: |
+          curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+      - name: Build
+        run: yarn run build:esm
+      - name: Run
+        run: |
+          NODE_ENV=production node benchmark/servers/ws8.mjs &
+          SERVER=ws8 ./k6 run benchmark/k6.mjs
+  fastify-websocket_ws8:
+    name: fastify-websocket_ws8
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+      - name: Install
+        run: yarn install --immutable
+      - name: Download k6
+        run: |
+          curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+      - name: Build
+        run: yarn run build:esm
+      - name: Run
+        run: |
+          NODE_ENV=production node benchmark/servers/fastify-websocket_ws8.mjs &
+          SERVER=fastify-websocket_ws8 ./k6 run benchmark/k6.mjs
+  legacy_ws7:
+    name: legacy_ws7
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+          cache: 'yarn'
+      - name: Install
+        run: yarn install --immutable
+      - name: Download k6
+        run: |
+          curl https://github.com/grafana/k6/releases/download/v0.34.1/k6-v0.34.1-linux-amd64.tar.gz -L | tar xvz --strip-components 1
+      - name: Build
+        run: yarn run build:esm
+      - name: Run
+        run: |
+          NODE_ENV=production node benchmark/servers/legacy_ws7.mjs &
+          LEGACY=1 SERVER=legacy_ws7 ./k6 run benchmark/k6.mjs

benchmark/k6.mjs

@@ -0,0 +1,148 @@
+import { check, fail } from 'k6';
+import ws from 'k6/ws';
+import { Counter, Trend } from 'k6/metrics';
+import { uuidv4 } from 'https://jslib.k6.io/k6-utils/1.0.0/index.js';
+import { ports } from './servers/ports.mjs';
+import { MessageType } from '../lib/common.mjs';
+
+if (!__ENV.SERVER) {
+  throw new Error('SERVER not specified.');
+}
+
+export const options = {
+  scenarios: {
+    query: {
+      executor: 'constant-vus',
+      exec: 'run',
+      vus: 20,
+    },
+    subscription: {
+      executor: 'constant-vus',
+      exec: 'run',
+      vus: 20,
+
+      env: { SUBSCRIPTION: '1' },
+    },
+  },
+};
+
+const duration = 10, // seconds
+  gracefulStop = 5; // seconds
+let i = 0;
+for (const [, scenario] of Object.entries(options.scenarios)) {
+  i++;
+
+  scenario.duration = duration + 's';
+  scenario.gracefulStop = gracefulStop + 's';
+  if (i > 1) {
+    scenario.startTime = (duration + gracefulStop) * (i - 1) + 's';
+  }
+}
+
+const scenarioMetrics = {};
+for (let scenario in options.scenarios) {
+  if (!options.scenarios[scenario].env) options.scenarios[scenario].env = {};
+  options.scenarios[scenario].env['SCENARIO'] = scenario;
+  if (!options.scenarios[scenario].tags) options.scenarios[scenario].tags = {};
+  options.scenarios[scenario].tags['SCENARIO'] = scenario;
+
+  scenarioMetrics[scenario] = {
+    runs: new Counter(`${scenario} - runs`),
+    opened: new Trend(`${scenario} - opened`, true),
+    subscribed: new Trend(`${scenario} - subscribed`, true),
+    completions: new Counter(`${scenario} - completions`),
+    completed: new Trend(`${scenario} - completed`, true),
+  };
+}
+
+export function run() {
+  const start = Date.now();
+
+  const metrics = scenarioMetrics[__ENV.SCENARIO];
+  metrics.runs.add(1);
+
+  let completed = false;
+  try {
+    ws.connect(
+      `ws://localhost:${ports[__ENV.SERVER]}/graphql`,
+      {
+        headers: {
+          'Sec-WebSocket-Protocol': __ENV.LEGACY
+            ? 'graphql-ws'
+            : 'graphql-transport-ws',
+        },
+      },
+      function (socket) {
+        // each run's socket can be open for no more than 3 seconds
+        socket.setTimeout(() => socket.close(), 3000);
+
+        socket.on('close', (code) => {
+          if (code !== 1000) throw null;
+        });
+
+        socket.on('open', () => {
+          metrics.opened.add(Date.now() - start);
+
+          socket.send(
+            JSON.stringify({
+              type: __ENV.LEGACY
+                ? 'connection_init'
+                : MessageType.ConnectionInit,
+            }),
+          );
+        });
+
+        let msgs = 0;
+        socket.on('message', (data) => {
+          msgs++;
+
+          if (msgs === 1) {
+            assertMessageType(
+              JSON.parse(data).type,
+              __ENV.LEGACY ? 'connection_ack' : MessageType.ConnectionAck,
+            );
+
+            socket.send(
+              JSON.stringify({
+                type: __ENV.LEGACY ? 'start' : MessageType.Subscribe,
+                id: uuidv4(),
+                payload: {
+                  query: __ENV.SUBSCRIPTION
+                    ? 'subscription { greetings }'
+                    : '{ hello }',
+                },
+              }),
+            );
+
+            metrics.subscribed.add(Date.now() - start);
+          } else if (__ENV.SUBSCRIPTION ? msgs > 1 && msgs <= 6 : msgs === 2) {
+            assertMessageType(
+              JSON.parse(data).type,
+              __ENV.LEGACY ? 'data' : MessageType.Next,
+            );
+          } else if (__ENV.SUBSCRIPTION ? msgs > 6 : msgs === 3) {
+            assertMessageType(
+              JSON.parse(data).type,
+              __ENV.LEGACY ? 'complete' : MessageType.Complete,
+            );
+
+            // we're done once completed
+            socket.close(1000);
+          } else fail(`Shouldn't have msgs ${msgs} messages`);
+        });
+      },
+    );
+    completed = true;
+    metrics.completed.add(Date.now() - start);
+    metrics.completions.add(1);
+  } catch (_err) {
+    // noop
+  }
+  check(0, { [`${__ENV.SCENARIO} - completed`]: () => completed });
+}
+
+function assertMessageType(got, expected) {
+  if (got !== expected) {
+    fail(`Expected ${expected} message, got ${got}`);
+  }
+}

benchmark/servers/fastify-websocket_ws8.mjs

@@ -0,0 +1,20 @@
+import Fastify from 'fastify';
+import fastifyWebsocket from 'fastify-websocket';
+import { ports } from './ports.mjs';
+import { makeHandler } from '../../lib/use/fastify-websocket.mjs';
+import { schema } from './schema.mjs';
+
+const fastify = Fastify();
+fastify.register(fastifyWebsocket);
+
+fastify.get('/graphql', { websocket: true }, makeHandler({ schema }));
+
+fastify.listen(ports['fastify-websocket_ws8'], (err) => {
+  if (err) {
+    fastify.log.error(err);
+    return process.exit(1);
+  }
+  console.log(
+    `fastify-websocket_ws8 - listening on port ${ports['fastify-websocket_ws8']}...`,
+  );
+});

benchmark/servers/index.mjs

@@ -0,0 +1,5 @@
+import './uWebSockets.mjs';
+import './ws8.mjs';
+import './ws7.mjs';
+import './fastify-websocket_ws8.mjs';
+import './legacy_ws7.mjs';

benchmark/servers/legacy_ws7.mjs

@@ -0,0 +1,53 @@
+import { createServer } from 'http';
+import { ports } from './ports.mjs';
+import { SubscriptionServer } from 'subscriptions-transport-ws';
+import { execute, subscribe } from 'graphql';
+import { schema } from './schema.mjs';
+
+const server = createServer((_req, res) => {
+  res.writeHead(404);
+  res.end();
+});
+
+SubscriptionServer.create(
+  {
+    schema,
+    execute: (
+      schema,
+      document,
+      rootValue,
+      contextValue,
+      variableValues,
+      operationName,
+    ) =>
+      execute({
+        schema,
+        document,
+        rootValue,
+        contextValue,
+        variableValues,
+        operationName,
+      }),
+    subscribe: (
+      schema,
+      document,
+      rootValue,
+      contextValue,
+      variableValues,
+      operationName,
+    ) =>
+      subscribe({
+        schema,
+        document,
+        rootValue,
+        contextValue,
+        variableValues,
+        operationName,
+      }),
+  },
+  { server, path: '/graphql' },
+);
+
+server.listen(ports.legacy_ws7, () => {
+  console.log(`legacy_ws7 - listening on port ${ports.legacy_ws7}...`);
+});

benchmark/servers/ports.mjs

@@ -0,0 +1,7 @@
+export const ports = {
+  ws8: 6540,
+  ws7: 6543,
+  uWebSockets: 6541,
+  legacy_ws7: 6542,
+  'fastify-websocket_ws8': 6544,
+};

benchmark/servers/schema.mjs

@@ -0,0 +1,27 @@
+import { GraphQLSchema, GraphQLObjectType, GraphQLString } from 'graphql';
+
+export const schema = new GraphQLSchema({
+  query: new GraphQLObjectType({
+    name: 'Query',
+    fields: {
+      hello: {
+        type: GraphQLString,
+        resolve: () => 'world',
+      },
+    },
+  }),
+  subscription: new GraphQLObjectType({
+    name: 'Subscription',
+    fields: {
+      greetings: {
+        type: GraphQLString,
+        subscribe: async function* () {
+          for (const hi of ['Hi', 'Bonjour', 'Hola', 'Ciao', 'Zdravo']) {
+            yield { greetings: hi };
+            await new Promise((resolve) => setImmediate(resolve));
+          }
+        },
+      },
+    },
+  }),
+});