From 2b93861c1b75642f348fcbc0e727733e0829bfae Mon Sep 17 00:00:00 2001 From: Brynley Llewellyn-Roux Date: Tue, 4 Feb 2025 08:39:30 +1100 Subject: [PATCH 1/9] feat: update CI --- .github/workflows/feature.yml | 19 +++++ .github/workflows/merge.yml | 144 ---------------------------------- .github/workflows/release.yml | 137 -------------------------------- .github/workflows/staging.yml | 27 +++++++ .github/workflows/tag.yml | 14 ++++ flake.lock | 37 +++------ flake.nix | 13 +-- 7 files changed, 77 insertions(+), 314 deletions(-) create mode 100644 .github/workflows/feature.yml delete mode 100644 .github/workflows/merge.yml delete mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/staging.yml create mode 100644 .github/workflows/tag.yml diff --git a/.github/workflows/feature.yml b/.github/workflows/feature.yml new file mode 100644 index 00000000..596cfd23 --- /dev/null +++ b/.github/workflows/feature.yml @@ -0,0 +1,19 @@ +name: "CI / Feature" + +on: + push: + branches: + - feature* + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + use-library-js-feature: + permissions: + contents: read + actions: write + checks: write + uses: MatrixAI/.github/.github/workflows/library-js-feature.yml@master diff --git a/.github/workflows/merge.yml b/.github/workflows/merge.yml deleted file mode 100644 index 72e79432..00000000 --- a/.github/workflows/merge.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: CI / Merge - -on: - push: - branches: - - staging - - feature* - -jobs: - check-lint: - name: "Check / Lint" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run linting - run: | - npm install - npm run lint - npm run lint-shell - - check-build: - name: "Check / Build" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run build - run: | - npm install - npm run build --verbose - - check-matrix: - name: "Check / Matrix" - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - id: set-matrix - run: | - files=$(find tests/* -maxdepth 0 -type d | sed 's/.*/"&"/' | paste -sd, -) - files=$files,$(find tests/* -maxdepth 0 -type f | grep -e "/*.test.ts" | sed 's/.*/"&"/' | paste -sd, -) - if [ -z "$files" ]; then - echo "matrix={\"shard\":[]}" >> $GITHUB_OUTPUT - else - echo "matrix={\"shard\":[$files]}" >> $GITHUB_OUTPUT - fi - - check-test: - name: "Check / Test" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{fromJson(needs.check-matrix.outputs.matrix)}} - needs: check-matrix - steps: - - uses: actions/checkout@v4 - - name: Set artifact name - run: echo "SLUG=$(echo ${{ matrix.shard }} | sed 's/[/.]/-/g')" >> $GITHUB_ENV - - name: Run tests - run: | - npm install - npm run test -- \ - --coverageReporters json \ - --coverage \ - "${{ matrix.shard }}" - mv tmp/coverage/coverage-final.json "tmp/coverage/${{ env.SLUG }}.json" - - uses: actions/upload-artifact@v4 - with: - name: coverage-artifacts-${{ env.SLUG }} - path: tmp/coverage/ - - check-coverage: - name: "Check / Coverage" - runs-on: ubuntu-latest - needs: check-test - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - pattern: coverage-artifacts-* - path: tmp/coverage/ - merge-multiple: true - - run: rm .npmrc - - name: Merge coverage results - run: npx nyc merge tmp/coverage/ tmp/coverage/cobertura-coverage.json - - uses: actions/upload-artifact@v4 - with: - name: cobertura-coverage - path: tmp/coverage/cobertura-coverage.json - - build-pull: - name: "Build / Pull Request" - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/staging' - steps: - - uses: actions/checkout@v4 - - name: Create pull request - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - run: | - gh pr create \ - --head staging \ - --base master \ - --title "ci: merge staging to master" \ - --body "This is an automatic PR generated by the CI/CD pipeline. This will be automatically fast-forward merged if successful." \ - --assignee "@me" \ - --no-maintainer-edit || true - printf "Pipeline Attempt on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - | gh pr comment staging \ - --body-file - \ - --repo "$GITHUB_REPOSITORY" - - integration-merge: - name: "Integration / Merge" - runs-on: ubuntu-latest - concurrency: - group: integration-merge - cancel-in-progress: true - needs: - - check-lint - - check-build - - check-test - - build-pull - if: github.ref == 'refs/heads/staging' - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GH_TOKEN }} - - name: Merge into master - env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} - GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} - GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} - GIT_COMMITTER_NAME: ${{ secrets.GIT_COMMITTER_NAME }} - run: | - printf "Pipeline Succeeded on $GITHUB_RUN_ID for $GITHUB_SHA\n\n$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ - | gh pr comment staging \ - --body-file - \ - --repo "$GITHUB_REPOSITORY" - git checkout master - git merge --ff-only "$GITHUB_SHA" - git push origin master diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index af70fa28..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,137 +0,0 @@ -name: CI / Release - -on: - push: - tags: - - v** - -jobs: - check-lint: - name: "Check / Lint" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run linting - run: | - npm install - npm run lint - npm run lint-shell - - check-build: - name: "Check / Build" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Run build - run: | - npm install - npm run build --verbose - - check-matrix: - name: "Check / Matrix" - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - id: set-matrix - run: | - files=$(find tests/* -maxdepth 0 -type d | sed 's/.*/"&"/' | paste -sd, -) - files=$files,$(find tests/* -maxdepth 0 -type f | grep -e "/*.test.ts" | sed 's/.*/"&"/' | paste -sd, -) - if [ -z "$files" ]; then - echo "matrix={\"shard\":[]}" >> $GITHUB_OUTPUT - else - echo "matrix={\"shard\":[$files]}" >> $GITHUB_OUTPUT - fi - - check-test: - name: "Check / Test" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: ${{fromJson(needs.check-matrix.outputs.matrix)}} - needs: check-matrix - steps: - - uses: actions/checkout@v4 - - name: Set artifact name - run: echo "SLUG=$(echo ${{ matrix.shard }} | sed 's/[/.]/-/g')" >> $GITHUB_ENV - - name: Run tests - run: | - npm install - npm run test -- \ - --coverageReporters json \ - --coverage \ - "${{ matrix.shard }}" - mv tmp/coverage/coverage-final.json "tmp/coverage/${{ env.SLUG }}.json" - - uses: actions/upload-artifact@v4 - with: - name: coverage-artifacts-${{ env.SLUG }} - path: tmp/coverage/ - - check-coverage: - name: "Check / Coverage" - runs-on: ubuntu-latest - needs: check-test - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - pattern: coverage-artifacts-* - path: tmp/coverage/ - merge-multiple: true - - run: rm .npmrc - - name: Merge coverage results - run: npx nyc merge tmp/coverage/ tmp/coverage/cobertura-coverage.json - - uses: actions/upload-artifact@v4 - with: - name: cobertura-coverage - path: tmp/coverage/cobertura-coverage.json - - build-prerelease: - name: "Build / Pre-release" - runs-on: ubuntu-latest - concurrency: - group: build-prerelease - cancel-in-progress: false - needs: - - check-lint - - check-build - - check-test - if: contains(github.ref, '-') - steps: - - uses: actions/checkout@v4 - - name: Run deployment - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc - echo 'Publishing library prerelease' - npm install - npm publish --tag prerelease --access public - rm -f ./.npmrc - - release-distribution: - name: "Release / Distribution" - runs-on: ubuntu-latest - concurrency: - group: release-distribution - cancel-in-progress: false - needs: - - check-lint - - check-build - - check-test - if: > - !contains(github.ref, '-') - steps: - - uses: actions/checkout@v4 - - name: Publish release - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ./.npmrc - echo 'Publishing library' - npm install - npm publish --access public - rm -f ./.npmrc - - diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml new file mode 100644 index 00000000..84413deb --- /dev/null +++ b/.github/workflows/staging.yml @@ -0,0 +1,27 @@ +name: "CI / Staging" + +on: + push: + branches: + - staging + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + use-library-js-staging: + permissions: + contents: write + actions: write + checks: write + pull-requests: write + uses: MatrixAI/.github/.github/workflows/library-js-staging.yml@master + secrets: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + GIT_AUTHOR_EMAIL: ${{ secrets.GIT_AUTHOR_EMAIL }} + GIT_AUTHOR_NAME: ${{ secrets.GIT_AUTHOR_NAME }} + GIT_COMMITTER_EMAIL: ${{ secrets.GIT_COMMITTER_EMAIL }} + GIT_COMMITTER_NAME: ${{ secrets.GIT_COMMITTER_NAME }} + diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml new file mode 100644 index 00000000..a8cc1cb2 --- /dev/null +++ b/.github/workflows/tag.yml @@ -0,0 +1,14 @@ +name: "CI / Tag" + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + +jobs: + use-library-js-tag: + permissions: + contents: read + actions: write + uses: MatrixAI/.github/.github/workflows/library-js-tag.yml@master diff --git a/flake.lock b/flake.lock index 0a36d4b6..5c7bdc98 100644 --- a/flake.lock +++ b/flake.lock @@ -5,11 +5,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1726560853, - "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -20,47 +20,30 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724223767, - "narHash": "sha256-Ifph01gDo4i4B50rpeqmhta/BYX2LQwXby4+BjlLqLM=", + "lastModified": 1736139540, + "narHash": "sha256-39Iclrd+9tPLmvuFVyoG63WnHZJ9kCOC6eRytRYLAWw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3f33387a5c85d94b305062a4f97d5b2899094efa", + "rev": "8ab83a21276434aaf44969b8dd0bc0e65b97a240", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "3f33387a5c85d94b305062a4f97d5b2899094efa", + "rev": "8ab83a21276434aaf44969b8dd0bc0e65b97a240", "type": "github" } }, "nixpkgs-matrix": { - "inputs": { - "nixpkgs-matrix": "nixpkgs-matrix_2" - }, - "locked": { - "lastModified": 1726032232, - "narHash": "sha256-zHREsuK+aCxfcHUxxLKTEUx156l7MRAPZAlRdIQtjHE=", - "owner": "MatrixAI", - "repo": "nixpkgs-matrix-private", - "rev": "b91eedf7fdeea8c0789e05193634356f5b4f408e", - "type": "github" - }, - "original": { - "id": "nixpkgs-matrix-private", - "type": "indirect" - } - }, - "nixpkgs-matrix_2": { "inputs": { "nixpkgs": "nixpkgs" }, "locked": { - "lastModified": 1726018991, - "narHash": "sha256-/CuJ75h6NW0UR0Nv/4XCCYhNsVTDn2ezCepRN26eMbM=", + "lastModified": 1736140072, + "narHash": "sha256-MgtcAA+xPldS0WlV16TjJ0qgFzGvKuGM9p+nPUxpUoA=", "owner": "MatrixAI", "repo": "nixpkgs-matrix", - "rev": "e0cff071d16c8b601558d4a597e9d5d39a0db7af", + "rev": "029084026bc4a35bce81bac898aa695f41993e18", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 8152a999..26c10260 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ inputs = { nixpkgs-matrix = { type = "indirect"; - id = "nixpkgs-matrix-private"; + id = "nixpkgs-matrix"; }; flake-utils.url = "github:numtide/flake-utils"; }; @@ -11,17 +11,15 @@ flake-utils.lib.eachDefaultSystem (system: let pkgs = nixpkgs-matrix.legacyPackages.${system}; - shell = { ci ? false }: with pkgs; - mkShell { + pkgs.mkShell { nativeBuildInputs = [ nodejs_20 shellcheck gitAndTools.gh ]; - NIX_DONT_SET_RPATH = true; - NIX_NO_SELF_RPATH = true; + PKG_IGNORE_TAG = 1; shellHook = '' echo "Entering $(npm pkg get name)" set -o allexport - . ./.env + . <(polykey secrets env js-ws) set +o allexport set -v ${lib.optionalString ci '' @@ -31,8 +29,11 @@ shopt -s inherit_errexit ''} mkdir --parents "$(pwd)/tmp" + export PATH="$(pwd)/dist/bin:$(npm root)/.bin:$PATH" + npm install --ignore-scripts + set +v ''; }; From 0ceb16b88e580105a5496d540a60dcb0b11a125a Mon Sep 17 00:00:00 2001 From: Brynley Llewellyn-Roux Date: Wed, 5 Feb 2025 13:12:18 +1100 Subject: [PATCH 2/9] fix: add `NPM_TOKEN` secret --- .github/workflows/tag.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml index a8cc1cb2..8965cafc 100644 --- a/.github/workflows/tag.yml +++ b/.github/workflows/tag.yml @@ -12,3 +12,5 @@ jobs: contents: read actions: write uses: MatrixAI/.github/.github/workflows/library-js-tag.yml@master + secrets: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} From 43b218824ca12f6b71f52257779720b1e36cd495 Mon Sep 17 00:00:00 2001 From: Brynley Llewellyn-Roux Date: Wed, 5 Feb 2025 13:12:51 +1100 Subject: [PATCH 3/9] 2.0.3 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 31ea3a55..faef1ae7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@matrixai/ws", - "version": "2.0.2", + "version": "2.0.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@matrixai/ws", - "version": "2.0.2", + "version": "2.0.3", "license": "Apache-2.0", "dependencies": { "@matrixai/async-cancellable": "^2.0.0", diff --git a/package.json b/package.json index 1a28f4ee..c70995b8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@matrixai/ws", - "version": "2.0.2", + "version": "2.0.3", "author": "Matrix AI", "contributors": [ { From 48250b3550e2aab45a108b96dde87e5c4f934bf8 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 28 Mar 2025 10:22:15 +1100 Subject: [PATCH 4/9] bench: updating benchmark structure --- .../baseline_tcp => }/baseline_tcp_1KiB.ts | 7 +- .../baseline_websocket_1KiB.ts | 9 +- .../connection => }/connection_1KiB.ts | 15 +-- benches/index.ts | 59 +++++----- benches/{suites/stream => }/stream_1KiB.ts | 16 +-- benches/utils.ts | 103 ------------------ benches/utils/index.ts | 1 + benches/utils/utils.ts | 66 +++++++++++ 8 files changed, 123 insertions(+), 153 deletions(-) rename benches/{suites/baseline_tcp => }/baseline_tcp_1KiB.ts (88%) rename benches/{suites/baseline_websocket => }/baseline_websocket_1KiB.ts (87%) rename benches/{suites/connection => }/connection_1KiB.ts (82%) rename benches/{suites/stream => }/stream_1KiB.ts (86%) delete mode 100644 benches/utils.ts create mode 100644 benches/utils/index.ts create mode 100644 benches/utils/utils.ts diff --git a/benches/suites/baseline_tcp/baseline_tcp_1KiB.ts b/benches/baseline_tcp_1KiB.ts similarity index 88% rename from benches/suites/baseline_tcp/baseline_tcp_1KiB.ts rename to benches/baseline_tcp_1KiB.ts index d5e98bf4..6b2d7b71 100644 --- a/benches/suites/baseline_tcp/baseline_tcp_1KiB.ts +++ b/benches/baseline_tcp_1KiB.ts @@ -1,8 +1,9 @@ -import type { Host } from '../../../src/types.js'; +import type { Host } from '#types.js'; import * as net from 'net'; +import path from 'node:path'; import url from 'node:url'; import b from 'benny'; -import { suiteCommon, summaryName } from '../../utils.js'; +import { suiteCommon } from './utils/utils.js'; import { promise } from '#utils.js'; const filePath = url.fileURLToPath(import.meta.url); @@ -38,7 +39,7 @@ async function main() { // Running benchmark const summary = await b.suite( - summaryName(filePath), + path.basename(filePath, path.extname(filePath)), b.add('send 1KiB of data over tcp', async () => { const prom = promise(); client.write(data1KiB, () => { diff --git a/benches/suites/baseline_websocket/baseline_websocket_1KiB.ts b/benches/baseline_websocket_1KiB.ts similarity index 87% rename from benches/suites/baseline_websocket/baseline_websocket_1KiB.ts rename to benches/baseline_websocket_1KiB.ts index a9e92d15..74b8bfaf 100644 --- a/benches/suites/baseline_websocket/baseline_websocket_1KiB.ts +++ b/benches/baseline_websocket_1KiB.ts @@ -1,11 +1,12 @@ -import type { Host } from '../../../src/types.js'; +import type { Host } from '#types.js'; import type { AddressInfo } from 'net'; import * as https from 'node:https'; +import path from 'node:path'; import url from 'node:url'; import b from 'benny'; import * as ws from 'ws'; -import { suiteCommon, summaryName } from '../../utils.js'; -import * as testsUtils from '../../../tests/utils.js'; +import { suiteCommon } from './utils/utils.js'; +import * as testsUtils from '../tests/utils.js'; import { promise } from '#utils.js'; const filePath = url.fileURLToPath(import.meta.url); @@ -42,7 +43,7 @@ async function main() { // Running benchmark const summary = await b.suite( - summaryName(filePath), + path.basename(filePath, path.extname(filePath)), b.add('send 1KiB of data over ws', async () => { const sendProm = promise(); client.send(data1KiB, { binary: true }, () => { diff --git a/benches/suites/connection/connection_1KiB.ts b/benches/connection_1KiB.ts similarity index 82% rename from benches/suites/connection/connection_1KiB.ts rename to benches/connection_1KiB.ts index 92820834..aef7ac1a 100644 --- a/benches/suites/connection/connection_1KiB.ts +++ b/benches/connection_1KiB.ts @@ -1,12 +1,13 @@ -import type { Host } from '../../../src/types.js'; +import type { Host } from '#types.js'; +import path from 'node:path'; import url from 'node:url'; import b from 'benny'; import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; -import { suiteCommon, summaryName } from '../../utils.js'; -import * as events from '../../../src/events.js'; -import * as testsUtils from '../../../tests/utils.js'; -import WebSocketServer from '../../../src/WebSocketServer.js'; -import WebSocketClient from '../../../src/WebSocketClient.js'; +import { suiteCommon } from './utils/utils.js'; +import * as testsUtils from '../tests/utils.js'; +import * as events from '#events.js'; +import WebSocketServer from '#WebSocketServer.js'; +import WebSocketClient from '#WebSocketClient.js'; const filePath = url.fileURLToPath(import.meta.url); @@ -51,7 +52,7 @@ async function main() { // Running benchmark const summary = await b.suite( - summaryName(filePath), + path.basename(filePath, path.extname(filePath)), b.add('send 1KiB of data over connection', async () => { // @ts-ignore: protected property await client.connection.send(data1KiB); diff --git a/benches/index.ts b/benches/index.ts index 786fb5bc..60bece7c 100644 --- a/benches/index.ts +++ b/benches/index.ts @@ -1,50 +1,51 @@ -#!/usr/bin/env ts-node +#!/usr/bin/env tsx -import type { Summary } from 'benny/lib/internal/common-types.js'; import fs from 'node:fs'; import path from 'node:path'; import url from 'node:url'; import si from 'systeminformation'; -import { fsWalk, resultsPath, suitesPath } from './utils.js'; - -const projectPath = path.dirname(url.fileURLToPath(import.meta.url)); +import { benchesPath } from './utils/utils.js'; +import baseline_tcp_1KiB from './baseline_tcp_1KiB.js'; +import baseline_websocket_1KiB from './baseline_websocket_1KiB.js'; +import connection_1KiB from './connection_1KiB.js'; +import stream_1KiB from './stream_1KiB.js'; async function main(): Promise { - await fs.promises.mkdir(path.join(projectPath, 'results'), { + await fs.promises.mkdir(path.join(benchesPath, 'results'), { recursive: true, }); - // Running all suites - for await (const suitePath of fsWalk(suitesPath)) { - // Skip over non-ts and non-js files - const ext = path.extname(suitePath); - if (ext !== '.ts' && ext !== '.js') { - continue; - } - const suite: () => Promise = (await import(suitePath)).default; - await suite(); - } - // Concatenating metrics - const metricsPath = path.join(resultsPath, 'metrics.txt'); + await baseline_tcp_1KiB(); + await baseline_websocket_1KiB(); + await connection_1KiB(); + await stream_1KiB(); + const resultFilenames = await fs.promises.readdir( + path.join(benchesPath, 'results'), + ); + const metricsFile = await fs.promises.open( + path.join(benchesPath, 'results', 'metrics.txt'), + 'w', + ); let concatenating = false; - for await (const metricPath of fsWalk(resultsPath)) { - // Skip over non-metrics files - if (!metricPath.endsWith('_metrics.txt')) { - continue; - } - const metricData = await fs.promises.readFile(metricPath); - if (concatenating) { - await fs.promises.appendFile(metricsPath, '\n'); + for (const resultFilename of resultFilenames) { + if (/.+_metrics\.txt$/.test(resultFilename)) { + const metricsData = await fs.promises.readFile( + path.join(benchesPath, 'results', resultFilename), + ); + if (concatenating) { + await metricsFile.write('\n'); + } + await metricsFile.write(metricsData); + concatenating = true; } - await fs.promises.appendFile(metricsPath, metricData); - concatenating = true; } + await metricsFile.close(); const systemData = await si.get({ cpu: '*', osInfo: 'platform, distro, release, kernel, arch', system: 'model, manufacturer', }); await fs.promises.writeFile( - path.join(projectPath, 'results', 'system.json'), + path.join(benchesPath, 'results', 'system.json'), JSON.stringify(systemData, null, 2), ); } diff --git a/benches/suites/stream/stream_1KiB.ts b/benches/stream_1KiB.ts similarity index 86% rename from benches/suites/stream/stream_1KiB.ts rename to benches/stream_1KiB.ts index b3f42706..0b08bd7a 100644 --- a/benches/suites/stream/stream_1KiB.ts +++ b/benches/stream_1KiB.ts @@ -1,12 +1,13 @@ -import type { Host } from '../../../src/types.js'; +import type { Host } from '#types.js'; +import path from 'node:path'; import url from 'node:url'; import b from 'benny'; import Logger, { formatting, LogLevel, StreamHandler } from '@matrixai/logger'; -import { suiteCommon, summaryName } from '../../utils.js'; -import * as events from '../../../src/events.js'; -import * as testsUtils from '../../../tests/utils.js'; -import WebSocketServer from '../../../src/WebSocketServer.js'; -import WebSocketClient from '../../../src/WebSocketClient.js'; +import { suiteCommon } from './utils/utils.js'; +import * as testsUtils from '../tests/utils.js'; +import * as events from '#events.js'; +import WebSocketServer from '#WebSocketServer.js'; +import WebSocketClient from '#WebSocketClient.js'; const filePath = url.fileURLToPath(import.meta.url); @@ -77,7 +78,7 @@ async function main() { // Running benchmark const summary = await b.suite( - summaryName(filePath), + path.basename(filePath, path.extname(filePath)), b.add('send 1KiB of data over stream', async () => { await writer.write(data1KiB); }), @@ -95,4 +96,5 @@ if (import.meta.url.startsWith('file:')) { void main(); } } + export default main; diff --git a/benches/utils.ts b/benches/utils.ts deleted file mode 100644 index 97d93501..00000000 --- a/benches/utils.ts +++ /dev/null @@ -1,103 +0,0 @@ -import fs from 'node:fs'; -import path from 'node:path'; -import url from 'node:url'; -import b from 'benny'; -import { codeBlock } from 'common-tags'; -import packageJson from '../package.json'; - -const projectPath = path.dirname(url.fileURLToPath(import.meta.url)); - -const suitesPath = path.join(projectPath, 'suites'); -const resultsPath = path.join(projectPath, 'results'); - -function summaryName(suitePath: string) { - return path - .relative(suitesPath, suitePath) - .replace(/\.[^.]*$/, '') - .replace(/\//g, '.'); -} - -const suiteCommon = [ - b.cycle(), - b.complete(), - b.save({ - file: (summary) => { - // Replace dots with slashes - const relativePath = summary.name.replace(/\./g, '/'); - // To `results/path/to/suite` - const resultPath = path.join(resultsPath, relativePath); - // This creates directory `results/path/to` - fs.mkdirSync(path.dirname(resultPath), { recursive: true }); - return relativePath; - }, - folder: resultsPath, - version: packageJson.version, - details: true, - }), - b.save({ - file: (summary) => { - // Replace dots with slashes - const relativePath = summary.name.replace(/\./g, '/'); - // To `results/path/to/suite` - const resultPath = path.join(resultsPath, relativePath); - // This creates directory `results/path/to` - fs.mkdirSync(path.dirname(resultPath), { recursive: true }); - return relativePath; - }, - folder: resultsPath, - version: packageJson.version, - format: 'chart.html', - }), - b.complete((summary) => { - // Replace dots with slashes - const relativePath = summary.name.replace(/\./g, '/'); - // To `results/path/to/suite_metrics.txt` - const resultPath = path.join(resultsPath, relativePath) + '_metrics.txt'; - // This creates directory `results/path/to` - fs.mkdirSync(path.dirname(resultPath), { recursive: true }); - fs.writeFileSync( - resultPath, - codeBlock` - # TYPE ${summary.name}_ops gauge - ${summary.results - .map( - (result) => - `${summary.name}_ops{name="${result.name}"} ${result.ops}`, - ) - .join('\n')} - - # TYPE ${summary.name}_margin gauge - ${summary.results - .map( - (result) => - `${summary.name}_margin{name="${result.name}"} ${result.margin}`, - ) - .join('\n')} - - # TYPE ${summary.name}_samples counter - ${summary.results - .map( - (result) => - `${summary.name}_samples{name="${result.name}"} ${result.samples}`, - ) - .join('\n')} - ` + '\n', - ); - // eslint-disable-next-line no-console - console.log('\nSaved to:', path.resolve(resultPath)); - }), -]; - -async function* fsWalk(dir: string): AsyncGenerator { - const dirents = await fs.promises.readdir(dir, { withFileTypes: true }); - for (const dirent of dirents) { - const res = path.resolve(dir, dirent.name); - if (dirent.isDirectory()) { - yield* fsWalk(res); - } else { - yield res; - } - } -} - -export { suitesPath, resultsPath, summaryName, suiteCommon, fsWalk }; diff --git a/benches/utils/index.ts b/benches/utils/index.ts new file mode 100644 index 00000000..9be8099f --- /dev/null +++ b/benches/utils/index.ts @@ -0,0 +1 @@ +export * from './utils.js'; diff --git a/benches/utils/utils.ts b/benches/utils/utils.ts new file mode 100644 index 00000000..ce8ec15b --- /dev/null +++ b/benches/utils/utils.ts @@ -0,0 +1,66 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import url from 'node:url'; +import b from 'benny'; +import { codeBlock } from 'common-tags'; +import packageJson from '../../package.json' assert { type: 'json' }; + +const benchesPath = path.dirname( + path.dirname(url.fileURLToPath(import.meta.url)), +); + +const suiteCommon = [ + b.cycle(), + b.complete(), + b.save({ + file: (summary) => summary.name, + folder: path.join(benchesPath, 'results'), + version: packageJson.version, + details: true, + }), + b.save({ + file: (summary) => summary.name, + folder: path.join(benchesPath, 'results'), + version: packageJson.version, + format: 'chart.html', + }), + b.complete((summary) => { + const filePath = path.join( + benchesPath, + 'results', + summary.name + '_metrics.txt', + ); + fs.writeFileSync( + filePath, + codeBlock` + # TYPE ${summary.name}_ops gauge + ${summary.results + .map( + (result) => + `${summary.name}_ops{name="${result.name}"} ${result.ops}`, + ) + .join('\n')} + + # TYPE ${summary.name}_margin gauge + ${summary.results + .map( + (result) => + `${summary.name}_margin{name="${result.name}"} ${result.margin}`, + ) + .join('\n')} + + # TYPE ${summary.name}_samples counter + ${summary.results + .map( + (result) => + `${summary.name}_samples{name="${result.name}"} ${result.samples}`, + ) + .join('\n')} + ` + '\n', + ); + // eslint-disable-next-line no-console + console.log('\nSaved to:', path.resolve(filePath)); + }), +]; + +export { benchesPath, suiteCommon }; From 996c425a8fc339bacd5c70f89cda62d17fa3a9a7 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Fri, 28 Mar 2025 10:24:47 +1100 Subject: [PATCH 5/9] bench: running benchmarks --- .../baseline_tcp/baseline_tcp_1KiB.json | 130 ------------------ .../baseline_tcp_1KiB_metrics.txt | 8 -- .../baseline_tcp_1KiB.chart.html | 14 +- benches/results/baseline_tcp_1KiB.json | 125 +++++++++++++++++ benches/results/baseline_tcp_1KiB_metrics.txt | 8 ++ .../baseline_websocket_1KiB.json | 129 ----------------- .../baseline_websocket_1KiB_metrics.txt | 8 -- .../baseline_websocket_1KiB.chart.html | 14 +- benches/results/baseline_websocket_1KiB.json | 121 ++++++++++++++++ .../baseline_websocket_1KiB_metrics.txt | 8 ++ .../results/connection/connection_1KiB.json | 129 ----------------- .../connection/connection_1KiB_metrics.txt | 8 -- .../connection_1KiB.chart.html | 14 +- benches/results/connection_1KiB.json | 118 ++++++++++++++++ benches/results/connection_1KiB_metrics.txt | 8 ++ benches/results/metrics.txt | 48 +++---- benches/results/stream/stream_1KiB.json | 91 ------------ .../results/stream/stream_1KiB_metrics.txt | 8 -- .../{stream => }/stream_1KiB.chart.html | 14 +- benches/results/stream_1KiB.json | 91 ++++++++++++ benches/results/stream_1KiB_metrics.txt | 8 ++ benches/results/system.json | 32 ++--- 22 files changed, 555 insertions(+), 579 deletions(-) delete mode 100644 benches/results/baseline_tcp/baseline_tcp_1KiB.json delete mode 100644 benches/results/baseline_tcp/baseline_tcp_1KiB_metrics.txt rename benches/results/{baseline_tcp => }/baseline_tcp_1KiB.chart.html (88%) create mode 100644 benches/results/baseline_tcp_1KiB.json create mode 100644 benches/results/baseline_tcp_1KiB_metrics.txt delete mode 100644 benches/results/baseline_websocket/baseline_websocket_1KiB.json delete mode 100644 benches/results/baseline_websocket/baseline_websocket_1KiB_metrics.txt rename benches/results/{baseline_websocket => }/baseline_websocket_1KiB.chart.html (87%) create mode 100644 benches/results/baseline_websocket_1KiB.json create mode 100644 benches/results/baseline_websocket_1KiB_metrics.txt delete mode 100644 benches/results/connection/connection_1KiB.json delete mode 100644 benches/results/connection/connection_1KiB_metrics.txt rename benches/results/{connection => }/connection_1KiB.chart.html (88%) create mode 100644 benches/results/connection_1KiB.json create mode 100644 benches/results/connection_1KiB_metrics.txt delete mode 100644 benches/results/stream/stream_1KiB.json delete mode 100644 benches/results/stream/stream_1KiB_metrics.txt rename benches/results/{stream => }/stream_1KiB.chart.html (89%) create mode 100644 benches/results/stream_1KiB.json create mode 100644 benches/results/stream_1KiB_metrics.txt diff --git a/benches/results/baseline_tcp/baseline_tcp_1KiB.json b/benches/results/baseline_tcp/baseline_tcp_1KiB.json deleted file mode 100644 index 021d7995..00000000 --- a/benches/results/baseline_tcp/baseline_tcp_1KiB.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "name": "baseline_tcp.baseline_tcp_1KiB", - "date": "2024-07-04T06:04:57.150Z", - "version": "1.2.0", - "results": [ - { - "name": "send 1KiB of data over tcp", - "ops": 1197455, - "margin": 1.05, - "options": { - "delay": 0.005, - "initCount": 1, - "minTime": 0.05, - "maxTime": 5, - "minSamples": 5 - }, - "samples": 86, - "promise": true, - "details": { - "min": 7.872292145296492e-7, - "max": 0.0000010375294008072029, - "mean": 8.351045249692836e-7, - "median": 8.267533784367521e-7, - "standardDeviation": 4.1459856304821e-8, - "marginOfError": 8.762633312651222e-9, - "relativeMarginOfError": 1.0492858140091537, - "standardErrorOfMean": 4.4707312819649095e-9, - "sampleVariance": 1.7189196848164058e-15, - "sampleResults": [ - 7.872292145296492e-7, - 7.892506364483079e-7, - 7.898806581806893e-7, - 7.941616268239677e-7, - 7.954494113172865e-7, - 7.97732427817448e-7, - 7.982046879850977e-7, - 7.985030735796336e-7, - 7.988529649177274e-7, - 7.995431542999069e-7, - 8.003239260303929e-7, - 8.017153406119765e-7, - 8.027002483700713e-7, - 8.041837929762077e-7, - 8.043388388699162e-7, - 8.066319155541757e-7, - 8.067974231605092e-7, - 8.072310773051848e-7, - 8.079307047500777e-7, - 8.080945824278174e-7, - 8.092207389009624e-7, - 8.095808506200768e-7, - 8.098783297112698e-7, - 8.100404687985097e-7, - 8.112343371623719e-7, - 8.117929524992238e-7, - 8.119039583980131e-7, - 8.121954517230674e-7, - 8.129409257293169e-7, - 8.129562868674325e-7, - 8.135287643588947e-7, - 8.151799441167339e-7, - 8.165870692331574e-7, - 8.179911983855946e-7, - 8.181729316070656e-7, - 8.182966780502949e-7, - 8.21397587075452e-7, - 8.216183017696368e-7, - 8.223921918658802e-7, - 8.250129307668426e-7, - 8.250372527313856e-7, - 8.256941167339335e-7, - 8.263199316982303e-7, - 8.271868251752739e-7, - 8.272235251239958e-7, - 8.27658817137535e-7, - 8.284299786623029e-7, - 8.28946848804719e-7, - 8.295028797869439e-7, - 8.311398789195901e-7, - 8.31249394597951e-7, - 8.314472252972036e-7, - 8.320649115207521e-7, - 8.325505588326607e-7, - 8.347442253958398e-7, - 8.352421106672443e-7, - 8.357226171996274e-7, - 8.358248864930772e-7, - 8.377067836075753e-7, - 8.393710183172928e-7, - 8.401221825520025e-7, - 8.429394683223436e-7, - 8.430268887070639e-7, - 8.431929896841872e-7, - 8.443838093759702e-7, - 8.452584911518162e-7, - 8.498279882024216e-7, - 8.514382570470552e-7, - 8.550274448928904e-7, - 8.568740608506675e-7, - 8.573342750698541e-7, - 8.605336830790458e-7, - 8.654612472124625e-7, - 8.709007075131155e-7, - 8.711753803166718e-7, - 8.730834523439925e-7, - 8.73613407626676e-7, - 8.8209537410742e-7, - 8.826423315740453e-7, - 8.826615648714123e-7, - 8.870778174479976e-7, - 8.978099180183215e-7, - 9.490311562464905e-7, - 9.495302629287828e-7, - 9.826761971095246e-7, - 0.0000010375294008072029 - ] - }, - "completed": true, - "percentSlower": 0 - } - ], - "fastest": { - "name": "send 1KiB of data over tcp", - "index": 0 - }, - "slowest": { - "name": "send 1KiB of data over tcp", - "index": 0 - } -} \ No newline at end of file diff --git a/benches/results/baseline_tcp/baseline_tcp_1KiB_metrics.txt b/benches/results/baseline_tcp/baseline_tcp_1KiB_metrics.txt deleted file mode 100644 index 4cd1449d..00000000 --- a/benches/results/baseline_tcp/baseline_tcp_1KiB_metrics.txt +++ /dev/null @@ -1,8 +0,0 @@ -# TYPE baseline_tcp.baseline_tcp_1KiB_ops gauge -baseline_tcp.baseline_tcp_1KiB_ops{name="send 1KiB of data over tcp"} 1197455 - -# TYPE baseline_tcp.baseline_tcp_1KiB_margin gauge -baseline_tcp.baseline_tcp_1KiB_margin{name="send 1KiB of data over tcp"} 1.05 - -# TYPE baseline_tcp.baseline_tcp_1KiB_samples counter -baseline_tcp.baseline_tcp_1KiB_samples{name="send 1KiB of data over tcp"} 86 diff --git a/benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html b/benches/results/baseline_tcp_1KiB.chart.html similarity index 88% rename from benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html rename to benches/results/baseline_tcp_1KiB.chart.html index abb59da0..ba877ed0 100644 --- a/benches/results/baseline_tcp/baseline_tcp_1KiB.chart.html +++ b/benches/results/baseline_tcp_1KiB.chart.html @@ -5,7 +5,7 @@ - baseline_tcp.baseline_tcp_1KiB + baseline_tcp_1KiB