From 84e4909b94a61376ceff656f9da710bafd58bb86 Mon Sep 17 00:00:00 2001 From: Brian Botha Date: Mon, 15 Nov 2021 17:06:59 +1100 Subject: [PATCH] Adding QA jobs and expanding test jobs for CI/CD. --- .gitignore | 1 + .gitlab-ci.yml | 305 +++++++++++++++++++++++-- .npmignore | 1 + default.nix | 4 +- package-lock.json | 400 +++++++++++---------------------- package.json | 9 +- src/agent/GRPCClientAgent.ts | 29 ++- src/client/GRPCClientClient.ts | 58 ++++- 8 files changed, 508 insertions(+), 299 deletions(-) diff --git a/.gitignore b/.gitignore index 18adc64cfb..322576fb4a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env* !.env.example /result* +/builds # Logs logs diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9cbf46403..75f92b4e67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,14 +1,17 @@ +default: + interruptible: true + variables: GIT_SUBMODULE_STRATEGY: recursive stages: - check - build + - quality - release -image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner - -lint: +.lint: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner stage: check interruptible: true script: @@ -18,19 +21,185 @@ lint: npm run lint; ' -test: +.test 1/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner stage: check interruptible: true script: - > nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' npm install; - npm run test; + npm run test -- tests/client; ' -nix-dry: +.test 2/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/agent; + ' + +.test 3/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/bin; + ' + +.test 4/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/bootstrap; + ' + +.test 5/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/network; + ' + +.test 6/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/vaults; + ' + +.test 7/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/workers; + ' + +.test 8/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/grpc; + ' + +.test 9/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner stage: check interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/http; + ' + +.test 10/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/GenericIdTypes.test.ts tests/index.test.ts tests/PolykeyAgent.test.ts tests/utils.test.ts; + ' + +.test 11/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/claims; + ' + +.test 12/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/gestalts; + ' + +.test 13/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/nodes; + ' + +.test 14/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/git; + ' + +.test 15/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/keys; + ' + +.test 16/16: + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + stage: check + interruptible: true + script: + - > + nix-shell -I nixpkgs=./pkgs.nix --packages nodejs --run ' + npm install; + npm run test -- tests/acl tests/discovery tests/lockfile tests/notifications tests/session tests/sigchain tests/identities; + ' + +.nix-dry: + stage: check + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner script: - nix-build -v -v --dry-run ./release.nix --attr application - nix-build -v -v --dry-run ./release.nix --attr docker @@ -40,31 +209,127 @@ nix-dry: nix: stage: build - interruptible: true + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner script: + - mkdir -p ./builds + # nix-specific application target - > - nix-build ./release.nix - --max-jobs $(nproc) - --attr application - --attr docker + build_application="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr application \ + )" + - > + nix-store --export $( \ + nix-store --query --requisites "$build_application" \ + ) | gzip > ./builds/js-polykey.closure.gz + # non-nix targets + - > + builds="$(nix-build \ + --max-jobs "$(nproc)" --cores "$(nproc)" \ + ./release.nix \ + --attr docker \ + --attr package.linux.x64.elf \ + --attr package.windows.x64.exe \ + --attr package.macos.x64.macho)" + - cp -r $builds ./builds/ + only: + - qa-testing + - master + artifacts: + paths: + - ./builds/ + +application run: + stage: quality + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + dependencies: + - nix + script: + - > + build_application="$( \ + gunzip -c ./builds/js-polykey.closure.gz | \ + nix-store --import | \ + tail -1 \ + )" + - echo password > testPassword + - $build_application/bin/polykey agent start -np ./nodePath --password-file ./testPassword --test + only: + - master + - qa-testing + +docker run: + stage: quality + image: docker:20.10.11 + dependencies: + - nix + services: + - docker:20.10.11-dind + variables: + DOCKER_TLS_CERTDIR: "/certs" + before_script: + - docker info + script: + - image="$(docker load --input ./builds/*docker* | cut -d' ' -f3)" + - docker run "$image" + only: + - qa-testing + - master + +linux run: + stage: quality + image: ubuntu:latest + dependencies: + - nix + script: + - echo password > testPassword + - for f in ./builds/*-linux-*; do "$f" agent start -np ./nodePath --password-file ./testPassword --test; done + only: + - qa-testing + - master + +windows run: + stage: quality + dependencies: + - nix + script: + - echo password > testPassword + - Get-ChildItem -File ./builds/*-win32-* | ForEach {& $_.FullName agent start -np ./nodePath --password-file ./testPassword --test} + tags: + - windows only: + - qa-testing - master -packages: +macos run: + stage: quality + image: macos-11-xcode-12 + dependencies: + - nix + script: + - echo password > testPassword + - for f in ./builds/*-macos-*; do "$f" agent start -np ./nodePath --password-file ./testPassword --test; done + only: + - qa-testing + - master + tags: + - shared-macos-amd64 + +.packages: stage: release - interruptible: true + image: registry.gitlab.com/matrixai/engineering/maintenance/gitlab-runner + dependencies: + - nix script: - > nix-shell -I nixpkgs=./pkgs.nix --packages git gitAndTools.gh --run ' - builds="$(nix-build \ - --max-jobs $(nproc) --cores $(nproc) \ - ./release.nix \ - --attr package.linux.x64.elf \ - --attr package.windows.x64.exe \ - --attr package.macos.x64.macho)"; commit="$(git rev-parse --short HEAD)"; gh release \ - create "$commit" $builds \ + create "$commit" \ + builds/*.closure.gz \ + builds/*-linux-* \ + builds/*-win32-* \ + builds/*-macos-* \ --title "Build-$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --prerelease \ --notes "" \ diff --git a/.npmignore b/.npmignore index afe09caa5e..4a1857eea4 100644 --- a/.npmignore +++ b/.npmignore @@ -13,3 +13,4 @@ /tmp /docs /benches +/builds diff --git a/default.nix b/default.nix index 478ceca03b..79e7317a44 100644 --- a/default.nix +++ b/default.nix @@ -12,7 +12,9 @@ let } '' mkdir -p $out/lib/node_modules/${utils.node2nixDev.packageName} - # copy only the dist + # copy the package.json + cp ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/package.json $out/lib/node_modules/${utils.node2nixDev.packageName}/ + # copy the dist cp -r ${utils.node2nixDev}/lib/node_modules/${utils.node2nixDev.packageName}/dist $out/lib/node_modules/${utils.node2nixDev.packageName}/ # copy over the production dependencies if [ -d "${utils.node2nixProd}/lib/node_modules" ]; then diff --git a/package-lock.json b/package-lock.json index a36b7bec9f..0a6a7fbf25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1582,15 +1582,15 @@ } }, "@matrixai/db": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@matrixai/db/-/db-1.1.0.tgz", - "integrity": "sha512-qoNbSbGzhZwjq+vXQcJ3Lk/z9NMNDlvSKd6wWe80i5ZMuVM+rbE8PenlWC4EXX77i+pfnV8rV51kz+V5FPt3Vg==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@matrixai/db/-/db-1.1.2.tgz", + "integrity": "sha512-wkVEEAJZaWS5Kbg6T/LcI6lS8AdWqszp8L1Dxmk7vwr1ihIkoIVQNSQ+FQryaFpor2eqh/wJaOKjDUpcHo+hEg==", "requires": { "@matrixai/logger": "^2.0.1", "@matrixai/workers": "^1.2.3", "abstract-leveldown": "^7.0.0", "async-mutex": "^0.3.1", - "level": "^6.0.1", + "level": "7.0.1", "levelup": "^5.0.1", "sublevel-prefixer": "^1.0.0", "subleveldown": "^5.0.1", @@ -1605,16 +1605,6 @@ "requires": { "tslib": "^2.3.1" } - }, - "level": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/level/-/level-6.0.1.tgz", - "integrity": "sha512-psRSqJZCsC/irNhfHzrVZbmPYXDcEYhA5TVNwr+V92jF44rbf86hqGp8fiT702FyiArScYIlPSBTDUASCVNSpw==", - "requires": { - "level-js": "^5.0.0", - "level-packager": "^5.1.0", - "leveldown": "^5.4.0" - } } } }, @@ -3150,48 +3140,12 @@ "dev": true }, "deferred-leveldown": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", - "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-7.0.0.tgz", + "integrity": "sha512-QKN8NtuS3BC6m0B8vAnBls44tX1WXAFATUsJlruyAYbZpysWV3siH6o/i3g9DCHauzodksO60bdj5NazNbjCmg==", "requires": { - "abstract-leveldown": "~6.2.1", + "abstract-leveldown": "^7.2.0", "inherits": "^2.0.3" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } - } } }, "define-properties": { @@ -3376,50 +3330,14 @@ "dev": true }, "encoding-down": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", - "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-7.1.0.tgz", + "integrity": "sha512-ky47X5jP84ryk5EQmvedQzELwVJPjCgXDQZGeb9F6r4PdChByCGHTBrVcF3h8ynKVJ1wVbkxTsDC8zBROPypgQ==", "requires": { - "abstract-leveldown": "^6.2.1", + "abstract-leveldown": "^7.2.0", "inherits": "^2.0.3", - "level-codec": "^9.0.0", - "level-errors": "^2.0.0" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.3.0.tgz", - "integrity": "sha512-TU5nlYgta8YrBMNpc9FwQzRbiXsj49gsALsXadbGHt9CROPzX5fB0rWDR5mtdpOOKa5XqRFpbj1QroPAoPzVjQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } - } + "level-codec": "^10.0.0", + "level-errors": "^3.0.0" } }, "encryptedfs": { @@ -6517,93 +6435,56 @@ "integrity": "sha1-eZllXoZGwX8In90YfRUNMyTVRRM=", "dev": true }, + "level": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/level/-/level-7.0.1.tgz", + "integrity": "sha512-w3E64+ALx2eZf8RV5JL4kIcE0BFAvQscRYd1yU4YVqZN9RGTQxXSvH202xvK15yZwFFxRXe60f13LJjcJ//I4Q==", + "requires": { + "level-js": "^6.1.0", + "level-packager": "^6.0.1", + "leveldown": "^6.1.0" + } + }, "level-codec": { - "version": "9.0.2", - "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", - "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-10.0.0.tgz", + "integrity": "sha512-QW3VteVNAp6c/LuV6nDjg7XDXx9XHK4abmQarxZmlRSDyXYk20UdaJTSX6yzVvQ4i0JyWSB7jert0DsyD/kk6g==", "requires": { - "buffer": "^5.6.0" - }, - "dependencies": { - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - } + "buffer": "^6.0.3" } }, "level-concat-iterator": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.0.0.tgz", - "integrity": "sha512-UHGiIdj+uiFQorOrURRvJF3Ei0uHc89ciM/aRi0qsWDV2f0HXypeXUPhJKL6DsONgSR76Pc0AI4sKYEYYRn2Dg==" - }, - "level-errors": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", - "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-3.1.0.tgz", + "integrity": "sha512-BWRCMHBxbIqPxJ8vHOvKUsaO0v1sLYZtjN3K2iZJsRBYtp+ONsY6Jfi6hy9K3+zolgQRryhIn2NRZjZnWJ9NmQ==", "requires": { - "errno": "~0.1.1" + "catering": "^2.1.0" } }, + "level-errors": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.1.tgz", + "integrity": "sha512-tqTL2DxzPDzpwl0iV5+rBCv65HWbHp6eutluHNcVIftKZlQN//b6GEnZDM2CvGZvzGYMwyPtYppYnydBQd2SMQ==" + }, "level-iterator-stream": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", - "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", + "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", "requires": { "inherits": "^2.0.4", - "readable-stream": "^3.4.0", - "xtend": "^4.0.2" + "readable-stream": "^3.4.0" } }, "level-js": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/level-js/-/level-js-5.0.2.tgz", - "integrity": "sha512-SnBIDo2pdO5VXh02ZmtAyPP6/+6YTJg2ibLtl9C34pWvmtMEmRTWpra+qO/hifkUtBTOtfx6S9vLDjBsBK4gRg==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/level-js/-/level-js-6.1.0.tgz", + "integrity": "sha512-i7mPtkZm68aewfv0FnIUWvFUFfoyzIvVKnUmuQGrelEkP72vSPTaA1SGneWWoCV5KZJG4wlzbJLp1WxVNGuc6A==", "requires": { - "abstract-leveldown": "~6.2.3", - "buffer": "^5.5.0", + "abstract-leveldown": "^7.2.0", + "buffer": "^6.0.3", "inherits": "^2.0.3", - "ltgt": "^2.1.2" - }, - "dependencies": { - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } - } + "ltgt": "^2.1.2", + "run-parallel-limit": "^1.1.0" } }, "level-option-wrap": { @@ -6615,84 +6496,33 @@ } }, "level-packager": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-5.1.1.tgz", - "integrity": "sha512-HMwMaQPlTC1IlcwT3+swhqf/NUO+ZhXVz6TY1zZIIZlIR0YSn8GtAAWmIvKjNY16ZkEg/JcpAuQskxsXqC0yOQ==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/level-packager/-/level-packager-6.0.1.tgz", + "integrity": "sha512-8Ezr0XM6hmAwqX9uu8IGzGNkWz/9doyPA8Oo9/D7qcMI6meJC+XhIbNYHukJhIn8OGdlzQs/JPcL9B8lA2F6EQ==", "requires": { - "encoding-down": "^6.3.0", - "levelup": "^4.3.2" - }, - "dependencies": { - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } - }, - "levelup": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/levelup/-/levelup-4.4.0.tgz", - "integrity": "sha512-94++VFO3qN95cM/d6eBXvd894oJE0w3cInq9USsyQzzoJxmiYzPAocNcuGCPGGjoXqDVJcr3C1jzt1TSjyaiLQ==", - "requires": { - "deferred-leveldown": "~5.3.0", - "level-errors": "~2.0.0", - "level-iterator-stream": "~4.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - } + "encoding-down": "^7.1.0", + "levelup": "^5.1.1" } }, "level-supports": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.0.2.tgz", - "integrity": "sha512-dU1W7OnntoCXeNfy9c93K7KEoGNsuP+zZLbUQrIbBzhdZ75U0h8GEcioqmJc1QpYVORyFfeR+kyFeNx2N4t7lg==" + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-2.1.0.tgz", + "integrity": "sha512-E486g1NCjW5cF78KGPrMDRBYzPuueMZ6VBXHT6gC7A8UYWGiM14fGgp+s/L1oFfDWSPV/+SFkYCmZ0SiESkRKA==" }, "leveldown": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-5.6.0.tgz", - "integrity": "sha512-iB8O/7Db9lPaITU1aA2txU/cBEXAt4vWwKQRrrWuS6XDgbP4QZGj9BL2aNbwb002atoQ/lIotJkfyzz+ygQnUQ==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/leveldown/-/leveldown-6.1.0.tgz", + "integrity": "sha512-8C7oJDT44JXxh04aSSsfcMI8YiaGRhOFI9/pMEL7nWJLVsWajDPTRxsSHTM2WcTVY5nXM+SuRHzPPi0GbnDX+w==", "requires": { - "abstract-leveldown": "~6.2.1", + "abstract-leveldown": "^7.2.0", "napi-macros": "~2.0.0", - "node-gyp-build": "~4.1.0" + "node-gyp-build": "^4.3.0" }, "dependencies": { - "abstract-leveldown": { - "version": "6.2.3", - "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", - "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", - "requires": { - "buffer": "^5.5.0", - "immediate": "^3.2.3", - "level-concat-iterator": "~2.0.0", - "level-supports": "~1.0.0", - "xtend": "~4.0.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, - "level-concat-iterator": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", - "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" - }, - "level-supports": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", - "integrity": "sha512-rXM7GYnW8gsl1vedTJIbzOrRv85c/2uCMpiiCzO2fndd06U/kUXEEU9evYn4zFggBOg36IsBW8LzqIpETwwQzg==", - "requires": { - "xtend": "^4.0.2" - } + "node-gyp-build": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.3.0.tgz", + "integrity": "sha512-iWjXZvmboq0ja1pUGULQBexmxq8CV4xBhX7VDOTbL7ZR4FOowwY/VOtRxBN/yKxmdGoIp4j5ysNT4u3S2pDQ3Q==" } } }, @@ -6707,31 +6537,6 @@ "level-iterator-stream": "^5.0.0", "level-supports": "^2.0.1", "queue-microtask": "^1.2.3" - }, - "dependencies": { - "deferred-leveldown": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-7.0.0.tgz", - "integrity": "sha512-QKN8NtuS3BC6m0B8vAnBls44tX1WXAFATUsJlruyAYbZpysWV3siH6o/i3g9DCHauzodksO60bdj5NazNbjCmg==", - "requires": { - "abstract-leveldown": "^7.2.0", - "inherits": "^2.0.3" - } - }, - "level-errors": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-3.0.1.tgz", - "integrity": "sha512-tqTL2DxzPDzpwl0iV5+rBCv65HWbHp6eutluHNcVIftKZlQN//b6GEnZDM2CvGZvzGYMwyPtYppYnydBQd2SMQ==" - }, - "level-iterator-stream": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-5.0.0.tgz", - "integrity": "sha512-wnb1+o+CVFUDdiSMR/ZymE2prPs3cjVLlXuDeSq9Zb8o032XrabGEXcTCsBxprAtseO3qvFeGzh6406z9sOTRA==", - "requires": { - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - } } }, "leven": { @@ -7146,9 +6951,10 @@ "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" }, "node-gyp-build": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.1.1.tgz", - "integrity": "sha512-dSq1xmcPDKPZ2EED2S6zw/b9NKsqzXRE6dVr8TVQnI3FJOTteUMuqF3Qqs6LZg+mLGYJWqQzMbIjMtJqTv87nQ==" + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.2.3.tgz", + "integrity": "sha512-MN6ZpzmfNCRM+3t57PTJHgHyw/h4OWnZ6mR8P5j/uZtqQr46RRuDE/P+g3n0YR/AiYXeWixZZzaip77gdICfRg==", + "dev": true }, "node-int64": { "version": "0.4.0", @@ -8158,6 +7964,14 @@ "queue-microtask": "^1.2.2" } }, + "run-parallel-limit": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/run-parallel-limit/-/run-parallel-limit-1.1.0.tgz", + "integrity": "sha512-jJA7irRNM91jaKc3Hcl1npHsFLOXOoTkPCUL1JEa1R82O2miplXXRaGdjW/KM/98YQWDhJLiSs793CnXfblJUw==", + "requires": { + "queue-microtask": "^1.2.2" + } + }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -8877,11 +8691,71 @@ "ieee754": "^1.1.13" } }, + "deferred-leveldown": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/deferred-leveldown/-/deferred-leveldown-5.3.0.tgz", + "integrity": "sha512-a59VOT+oDy7vtAbLRCZwWgxu2BaCfd5Hk7wxJd48ei7I+nsg8Orlb9CLG0PMZienk9BSUKgeAqkO2+Lw+1+Ukw==", + "requires": { + "abstract-leveldown": "~6.2.1", + "inherits": "^2.0.3" + }, + "dependencies": { + "abstract-leveldown": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", + "integrity": "sha512-BsLm5vFMRUrrLeCcRc+G0t2qOaTzpoJQLOubq2XM72eNpjF5UdU5o/5NvlNhx95XHcAvcl8OMXr4mlg/fRgUXQ==", + "requires": { + "buffer": "^5.5.0", + "immediate": "^3.2.3", + "level-concat-iterator": "~2.0.0", + "level-supports": "~1.0.0", + "xtend": "~4.0.0" + } + } + } + }, + "encoding-down": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/encoding-down/-/encoding-down-6.3.0.tgz", + "integrity": "sha512-QKrV0iKR6MZVJV08QY0wp1e7vF6QbhnbQhb07bwpEyuz4uZiZgPlEGdkCROuFkUwdxlFaiPIhjyarH1ee/3vhw==", + "requires": { + "abstract-leveldown": "^6.2.1", + "inherits": "^2.0.3", + "level-codec": "^9.0.0", + "level-errors": "^2.0.0" + } + }, + "level-codec": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/level-codec/-/level-codec-9.0.2.tgz", + "integrity": "sha512-UyIwNb1lJBChJnGfjmO0OR+ezh2iVu1Kas3nvBS/BzGnx79dv6g7unpKIDNPMhfdTEGoc7mC8uAu51XEtX+FHQ==", + "requires": { + "buffer": "^5.6.0" + } + }, "level-concat-iterator": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/level-concat-iterator/-/level-concat-iterator-2.0.1.tgz", "integrity": "sha512-OTKKOqeav2QWcERMJR7IS9CUo1sHnke2C0gkSmcR7QuEtFNLLzHQAvnMw8ykvEcv0Qtkg0p7FOwP1v9e5Smdcw==" }, + "level-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/level-errors/-/level-errors-2.0.1.tgz", + "integrity": "sha512-UVprBJXite4gPS+3VznfgDSU8PTRuVX0NXwoWW50KLxd2yw4Y1t2JUR5In1itQnudZqRMT9DlAM3Q//9NCjCFw==", + "requires": { + "errno": "~0.1.1" + } + }, + "level-iterator-stream": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-4.0.2.tgz", + "integrity": "sha512-ZSthfEqzGSOMWoUGhTXdX9jv26d32XJuHz/5YnuHZzH6wldfWMOVwI9TBtKcya4BKTyTt3XVA0A3cF3q5CY30Q==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "^3.4.0", + "xtend": "^4.0.2" + } + }, "level-supports": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/level-supports/-/level-supports-1.0.1.tgz", diff --git a/package.json b/package.json index 713f794513..3861cb3733 100644 --- a/package.json +++ b/package.json @@ -52,13 +52,15 @@ "assets": [ "node_modules/jose/**/*", "node_modules/utp-native/**/*", - "node_modules/leveldown/**/*" + "node_modules/leveldown/**/*", + "node_modules/fd-lock/**/*", + "dist/**/*.json" ], "scripts": "dist/workers/polykeyWorker.js" }, "scripts": { "build": "rm -r ./dist || true; tsc -p ./tsconfig.build.json", - "postbuild": "cp -fR src/proto dist", + "postbuild": "cp -fR src/proto dist; cp src/notifications/*.json dist/notifications/; cp src/claims/*.json dist/claims/", "ts-node": "ts-node --require tsconfig-paths/register", "test": "jest", "lint": "eslint '{src,tests}/**/*.{js,ts}'", @@ -70,7 +72,7 @@ "dependencies": { "@grpc/grpc-js": "1.3.7", "@matrixai/async-init": "^1.1.0", - "@matrixai/db": "^1.1.0", + "@matrixai/db": "^1.1.2", "@matrixai/id": "^2.1.0", "@matrixai/logger": "^2.1.0", "@matrixai/workers": "^1.2.3", @@ -102,6 +104,7 @@ "uuid": "^8.3.0" }, "devDependencies": { + "node-gyp-build": "4.2.3", "@babel/preset-env": "^7.13.10", "@types/cross-spawn": "^6.0.2", "@types/google-protobuf": "^3.7.4", diff --git a/src/agent/GRPCClientAgent.ts b/src/agent/GRPCClientAgent.ts index 25d5432a73..f66621b9a0 100644 --- a/src/agent/GRPCClientAgent.ts +++ b/src/agent/GRPCClientAgent.ts @@ -5,6 +5,9 @@ import type * as utilsPB from '../proto/js/polykey/v1/utils/utils_pb'; import type * as vaultsPB from '../proto/js/polykey/v1/vaults/vaults_pb'; import type * as nodesPB from '../proto/js/polykey/v1/nodes/nodes_pb'; import type * as notificationsPB from '../proto/js/polykey/v1/notifications/notifications_pb'; +import type { types as grpcTypes } from '../grpc'; +import type { ClientDuplexStream } from '@grpc/grpc-js'; +import type { ClientReadableStream } from '@grpc/grpc-js/build/src/call'; import Logger from '@matrixai/logger'; import { CreateDestroy, ready } from '@matrixai/async-init/dist/CreateDestroy'; import * as agentErrors from './errors'; @@ -77,7 +80,12 @@ class GRPCClientAgent extends GRPCClient { } @ready(new agentErrors.ErrorAgentClientDestroyed()) - public vaultsGitInfoGet(...args) { + public vaultsGitInfoGet( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.PackChunk, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsGitInfoGet, @@ -85,12 +93,19 @@ class GRPCClientAgent extends GRPCClient { } @ready(new agentErrors.ErrorAgentClientDestroyed()) - public vaultsGitPackGet(...args) { + public vaultsGitPackGet( + ...args + ): ClientDuplexStream { return this.client.vaultsGitPackGet(...args); } @ready(new agentErrors.ErrorAgentClientDestroyed()) - public vaultsScan(...args) { + public vaultsScan( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.Vault, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsScan, @@ -146,7 +161,13 @@ class GRPCClientAgent extends GRPCClient { } @ready(new agentErrors.ErrorAgentClientDestroyed()) - public nodesCrossSignClaim(...args) { + public nodesCrossSignClaim( + ...args + ): grpcTypes.AsyncGeneratorDuplexStreamClient< + nodesPB.CrossSign, + nodesPB.CrossSign, + ClientDuplexStream + > { return grpcUtils.promisifyDuplexStreamCall< nodesPB.CrossSign, nodesPB.CrossSign diff --git a/src/client/GRPCClientClient.ts b/src/client/GRPCClientClient.ts index 9beec1423b..644065842a 100644 --- a/src/client/GRPCClientClient.ts +++ b/src/client/GRPCClientClient.ts @@ -14,6 +14,8 @@ import type * as identitiesPB from '../proto/js/polykey/v1/identities/identities import type * as keysPB from '../proto/js/polykey/v1/keys/keys_pb'; import type * as permissionsPB from '../proto/js/polykey/v1/permissions/permissions_pb'; import type * as secretsPB from '../proto/js/polykey/v1/secrets/secrets_pb'; +import type { types as grpcTypes } from '../grpc'; +import type { ClientReadableStream } from '@grpc/grpc-js/build/src/call'; import { CreateDestroy, ready } from '@matrixai/async-init/dist/CreateDestroy'; import Logger from '@matrixai/logger'; import * as clientErrors from './errors'; @@ -126,7 +128,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public vaultsList(...args) { + public vaultsList( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.List, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsList, @@ -174,7 +181,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public vaultsScan(...args) { + public vaultsScan( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.List, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsScan, @@ -198,7 +210,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public vaultPermissions(...args) { + public vaultPermissions( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.Permission, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsPermissions, @@ -206,7 +223,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public vaultsSecretsList(...args) { + public vaultsSecretsList( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + secretsPB.Secret, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsSecretsList, @@ -286,7 +308,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public vaultsLog(...args) { + public vaultsLog( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + vaultsPB.LogEntry, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.vaultsLog, @@ -366,7 +393,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public keysCertsChainGet(...args) { + public keysCertsChainGet( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + keysPB.Certificate, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.keysCertsChainGet, @@ -374,7 +406,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public gestaltsGestaltList(...args) { + public gestaltsGestaltList( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + gestaltsPB.Gestalt, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.gestaltsGestaltList, @@ -526,7 +563,12 @@ class GRPCClientClient extends GRPCClient { } @ready(new clientErrors.ErrorClientClientDestroyed()) - public identitiesAuthenticate(...args) { + public identitiesAuthenticate( + ...args + ): grpcTypes.AsyncGeneratorReadableStreamClient< + identitiesPB.Provider, + ClientReadableStream + > { return grpcUtils.promisifyReadableStreamCall( this.client, this.client.identitiesAuthenticate,