From 17da3fa9495ea0948a4bdb6fd721b21d5cc6778a Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:24:58 +0100 Subject: [PATCH 01/85] Drop indexes for table sync_data_records that are not/little used (#1426) Used the query described in the following blog post to determine which indexes are not (or rarely) used http://www.databasesoup.com/2014/05/new-finding-unused-indexes-query.html --- ...213124728_drop_sync_data_records_indexes.cjs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs diff --git a/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs new file mode 100644 index 00000000000..1b6de2e0da8 --- /dev/null +++ b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs @@ -0,0 +1,17 @@ +exports.config = { transaction: false }; + +exports.up = function(knex) { + return knex.schema + .raw('DROP INDEX CONCURRENTLY created_at_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_data_hash_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_sync_job_id_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_external_is_deleted_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_external_deleted_at_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_data_hash_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_sync_job_id_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_external_is_deleted_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_external_deleted_at_index') + .raw('DROP INDEX CONCURRENTLY _nango_sync_jobs_deleted_index'); +}; + +exports.down = function(knex) { }; From e52bacbf9ef1ac4dcd989c414dad1e7bd8c2338c Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Thu, 14 Dec 2023 11:01:09 +0100 Subject: [PATCH 02/85] sdk: fix setMetadata argument type (#1428) --- packages/shared/lib/sdk/sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index 07f66dbb064..b4e0c637d59 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -326,7 +326,7 @@ export class NangoAction { return this.nango.getConnection(this.providerConfigKey as string, this.connectionId as string); } - public async setMetadata(metadata: Record): Promise> { + public async setMetadata(metadata: Record): Promise> { return this.nango.setMetadata(this.providerConfigKey as string, this.connectionId as string, metadata); } From ed689eccd4cd6f3d15925e494a5ced1259911f46 Mon Sep 17 00:00:00 2001 From: Robin Guldener Date: Thu, 14 Dec 2023 17:14:48 +0100 Subject: [PATCH 03/85] Improve freshdesk docs --- docs-v2/integrations/all/freshdesk.mdx | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs-v2/integrations/all/freshdesk.mdx b/docs-v2/integrations/all/freshdesk.mdx index a725c614187..977b55adccb 100644 --- a/docs-v2/integrations/all/freshdesk.mdx +++ b/docs-v2/integrations/all/freshdesk.mdx @@ -24,6 +24,25 @@ API configuration: [`freshdesk`](https://nango.dev/providers.yaml) Need help getting started? Get help in the [community](https://nango.dev/slack). +## Connection configuration in Nango + +Freshdesk requires a user specific subdomain for the API requests. + +You should request this from the user and pass it to Nango in the `nango.auth()` call: + +```js +nango.auth('freshdesk', '', { + params: { + subdomain: '' + }, + credentials: { + username: '', + password: 'x' // freshdesk asks you to leave "x" here + }}); +``` + +For more details, see the [docs here](/guides/advanced-auth#connection-configuration). + ## API gotchas - For Basic Auth, Freshdesk uses API key as a username and dummy characters as a password. From 1e0b394335b92b7e0a69578a964e58600c14d66b Mon Sep 17 00:00:00 2001 From: Robin Guldener Date: Thu, 14 Dec 2023 17:19:08 +0100 Subject: [PATCH 04/85] Add Freshdesk to support lists --- docs-v2/integrations/support.mdx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs-v2/integrations/support.mdx b/docs-v2/integrations/support.mdx index 4a64afc60d6..813f78cabf5 100644 --- a/docs-v2/integrations/support.mdx +++ b/docs-v2/integrations/support.mdx @@ -10,9 +10,12 @@ sidebarTitle: Support + + + From 32291a00f021e0a3a303e225052bfe2587632c87 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Fri, 15 Dec 2023 19:50:50 +0100 Subject: [PATCH 05/85] Add jobs and runner service (#1429) More info at https://www.notion.so/nangohq/Nov-2023-Secure-Code-Sandbox-759ad0b6941e4149af0ab2321d9e3ee5?pvs=4#5c513f40a15e4e688a459caf8b22735c --- package-lock.json | 1680 ++++++++++++++++- package.json | 11 +- .../lib/services/local-integration.service.ts | 5 +- packages/jobs/.gitignore | 3 + packages/jobs/Dockerfile | 20 + packages/jobs/lib/activities.ts | 354 ++++ packages/jobs/lib/app.ts | 32 + packages/jobs/lib/client.ts | 8 + packages/jobs/lib/integration.service.ts | 149 ++ packages/jobs/lib/models/worker.ts | 26 + packages/jobs/lib/runner/local.runner.ts | 55 + packages/jobs/lib/runner/render.runner.ts | 62 + packages/jobs/lib/runner/runner.ts | 29 + packages/jobs/lib/server.ts | 24 + packages/jobs/lib/temporal.ts | 66 + packages/jobs/lib/tracer.ts | 9 + packages/jobs/lib/workflows.ts | 46 + packages/jobs/nodemon.json | 7 + packages/jobs/package.json | 52 + packages/jobs/tsconfig.json | 9 + packages/runner/Dockerfile | 19 + packages/runner/lib/app.ts | 11 + packages/runner/lib/client.ts | 8 + packages/runner/lib/client.unit.test.ts | 43 + packages/runner/lib/exec.ts | 46 + packages/runner/lib/index.ts | 1 + packages/runner/lib/server.ts | 49 + packages/runner/nodemon.json | 6 + packages/runner/package.json | 30 + packages/runner/tsconfig.json | 9 + packages/shared/lib/models/Sync.ts | 4 +- packages/shared/lib/sdk/sync.ts | 3 +- .../shared/lib/services/sync/run.service.ts | 10 +- packages/worker/lib/integration.service.ts | 4 +- tsconfig.build.json | 6 +- tsconfig.json | 2 +- 36 files changed, 2796 insertions(+), 102 deletions(-) create mode 100644 packages/jobs/.gitignore create mode 100644 packages/jobs/Dockerfile create mode 100644 packages/jobs/lib/activities.ts create mode 100644 packages/jobs/lib/app.ts create mode 100644 packages/jobs/lib/client.ts create mode 100644 packages/jobs/lib/integration.service.ts create mode 100644 packages/jobs/lib/models/worker.ts create mode 100644 packages/jobs/lib/runner/local.runner.ts create mode 100644 packages/jobs/lib/runner/render.runner.ts create mode 100644 packages/jobs/lib/runner/runner.ts create mode 100644 packages/jobs/lib/server.ts create mode 100644 packages/jobs/lib/temporal.ts create mode 100644 packages/jobs/lib/tracer.ts create mode 100644 packages/jobs/lib/workflows.ts create mode 100644 packages/jobs/nodemon.json create mode 100644 packages/jobs/package.json create mode 100644 packages/jobs/tsconfig.json create mode 100644 packages/runner/Dockerfile create mode 100644 packages/runner/lib/app.ts create mode 100644 packages/runner/lib/client.ts create mode 100644 packages/runner/lib/client.unit.test.ts create mode 100644 packages/runner/lib/exec.ts create mode 100644 packages/runner/lib/index.ts create mode 100644 packages/runner/lib/server.ts create mode 100644 packages/runner/nodemon.json create mode 100644 packages/runner/package.json create mode 100644 packages/runner/tsconfig.json diff --git a/package-lock.json b/package-lock.json index 61d0622515a..287d2df090b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,9 @@ "packages/frontend", "packages/node-client", "packages/server", - "packages/worker" + "packages/worker", + "packages/runner", + "packages/jobs" ], "dependencies": { "@babel/parser": "^7.22.5", @@ -62,6 +64,19 @@ "node": ">=6.0.0" } }, + "node_modules/@apidevtools/openapi-schemas": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", + "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==", + "engines": { + "node": ">=10" + } + }, + "node_modules/@apidevtools/swagger-methods": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", + "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + }, "node_modules/@aws-crypto/crc32": { "version": "3.0.0", "license": "Apache-2.0", @@ -1270,8 +1285,9 @@ } }, "node_modules/@babel/highlight": { - "version": "7.22.20", - "license": "MIT", + "version": "7.23.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.23.4.tgz", + "integrity": "sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==", "dependencies": { "@babel/helper-validator-identifier": "^7.22.20", "chalk": "^2.4.2", @@ -2545,7 +2561,6 @@ }, "node_modules/@babel/runtime": { "version": "7.22.3", - "dev": true, "license": "MIT", "dependencies": { "regenerator-runtime": "^0.13.11" @@ -2805,9 +2820,9 @@ } }, "node_modules/@datadog/native-appsec": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-4.0.0.tgz", - "integrity": "sha512-myTguXJ3VQHS2E1ylNsSF1avNpDmq5t+K4Q47wdzeakGc3sDIDDyEbvuFTujl9c9wBIkup94O1mZj5DR37ajzA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-5.0.0.tgz", + "integrity": "sha512-Ks8a4L49N40w+TJjj2e9ncGssUIEjo4wnmUFjPBRvlLGuVj1VJLxCx7ztpd8eTycM5QQlzggCDOP6CMEVmeZbA==", "hasInstallScript": true, "dependencies": { "node-gyp-build": "^3.9.0" @@ -2869,9 +2884,9 @@ } }, "node_modules/@datadog/pprof": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-4.0.1.tgz", - "integrity": "sha512-TavqyiyQZOaUM9eQB07r8+K/T1CqKyOdsUGxpN79+BF+eOQBpTj/Cte6KdlhcUSKL3h5hSjC+vlgA7uW2qtVhA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-4.1.0.tgz", + "integrity": "sha512-g7EWI185nwSuFwlmnAGDPxbPsqe+ipOoDB2oP841WMNRaJBPRdg5J90c+6ucmyltuC9VpTrmzzqcachkOTzZEQ==", "hasInstallScript": true, "dependencies": { "delay": "^5.0.0", @@ -2947,6 +2962,11 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/@exodus/schemasafe": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@exodus/schemasafe/-/schemasafe-1.3.0.tgz", + "integrity": "sha512-5Aap/GaRupgNx/feGBwLLTVv8OQFfv3pq2lPRzPg9R+IOBnDgghTGW7l7EuVXOvg5cc/xSAlRW8rBrjIC3Nvqw==" + }, "node_modules/@grpc/grpc-js": { "version": "1.7.3", "license": "Apache-2.0", @@ -3063,6 +3083,14 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@humanwhocodes/momoa": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@humanwhocodes/momoa/-/momoa-2.0.4.tgz", + "integrity": "sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA==", + "engines": { + "node": ">=10.10.0" + } + }, "node_modules/@humanwhocodes/object-schema": { "version": "1.2.1", "dev": true, @@ -3373,10 +3401,23 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@jsdevtools/ono": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@jsdevtools/ono/-/ono-7.1.3.tgz", + "integrity": "sha512-4JQNk+3mVzK3xh2rqd6RB4J46qUR19azEHBneZyTZM+c456qOrbbM/5xcR8huNCCcbVt7+UmizG6GuUvPvKUYg==" + }, "node_modules/@nangohq/frontend": { "resolved": "packages/frontend", "link": true }, + "node_modules/@nangohq/nango-jobs": { + "resolved": "packages/jobs", + "link": true + }, + "node_modules/@nangohq/nango-runner": { + "resolved": "packages/runner", + "link": true + }, "node_modules/@nangohq/nango-server": { "resolved": "packages/server", "link": true @@ -3395,7 +3436,6 @@ }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -3407,7 +3447,6 @@ }, "node_modules/@nodelib/fs.stat": { "version": "2.0.5", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -3415,7 +3454,6 @@ }, "node_modules/@nodelib/fs.walk": { "version": "1.2.8", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -3769,6 +3807,223 @@ "version": "1.1.0", "license": "BSD-3-Clause" }, + "node_modules/@readme/better-ajv-errors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@readme/better-ajv-errors/-/better-ajv-errors-1.6.0.tgz", + "integrity": "sha512-9gO9rld84Jgu13kcbKRU+WHseNhaVt76wYMeRDGsUGYxwJtI3RmEJ9LY9dZCYQGI8eUZLuxb5qDja0nqklpFjQ==", + "dependencies": { + "@babel/code-frame": "^7.16.0", + "@babel/runtime": "^7.21.0", + "@humanwhocodes/momoa": "^2.0.3", + "chalk": "^4.1.2", + "json-to-ast": "^2.0.3", + "jsonpointer": "^5.0.0", + "leven": "^3.1.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "ajv": "4.11.8 - 8" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/@babel/code-frame": { + "version": "7.23.5", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.23.5.tgz", + "integrity": "sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==", + "dependencies": { + "@babel/highlight": "^7.23.4", + "chalk": "^2.4.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/@babel/code-frame/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, + "node_modules/@readme/better-ajv-errors/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/@readme/better-ajv-errors/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@readme/data-urls": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@readme/data-urls/-/data-urls-1.0.1.tgz", + "integrity": "sha512-FNP4ntG5rCgmrvQGoNH/Ljivc6jSWaaVeMuXneOyQ6oLuhm/NkysXJN3DnBrIsJUJbSae7qIs2QfPYnaropoHw==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@readme/http-status-codes": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@readme/http-status-codes/-/http-status-codes-7.2.0.tgz", + "integrity": "sha512-/dBh9qw3QhJYqlGwt2I+KUP/lQ6nytdCx3aq+GpMUhibLHF3O7fwoowNcTwlbnwtyJ+TJYTIIrp3oVUlRNx3fA==" + }, + "node_modules/@readme/json-schema-ref-parser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@readme/json-schema-ref-parser/-/json-schema-ref-parser-1.2.0.tgz", + "integrity": "sha512-Bt3QVovFSua4QmHa65EHUmh2xS0XJ3rgTEUPH998f4OW4VVJke3BuS16f+kM0ZLOGdvIrzrPRqwihuv5BAjtrA==", + "dependencies": { + "@jsdevtools/ono": "^7.1.3", + "@types/json-schema": "^7.0.6", + "call-me-maybe": "^1.0.1", + "js-yaml": "^4.1.0" + } + }, + "node_modules/@readme/oas-extensions": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/@readme/oas-extensions/-/oas-extensions-17.0.1.tgz", + "integrity": "sha512-PCU7WLz8TkbdxsiE4eQGvJYDYZQPiyLhXme3SvLboSmH+8G6AJPJ5OymzSAdlf5sXpSSoD2q3dTIou3Cb2DirQ==", + "deprecated": "The functionality for this library has been moved into `oas`.", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "oas": "^20.0.0" + } + }, + "node_modules/@readme/oas-to-har": { + "version": "20.1.1", + "resolved": "https://registry.npmjs.org/@readme/oas-to-har/-/oas-to-har-20.1.1.tgz", + "integrity": "sha512-rz8YpdZw+Jqrd8VQhQaYrzctkCAYdBldoQ5qDQyF9vGvq2lpA1yMvQPgKCJXfPGXH8Cm+NjLbunxnYabKQeKeA==", + "dependencies": { + "@readme/data-urls": "^1.0.1", + "@readme/oas-extensions": "^17.0.1", + "oas": "^20.5.0", + "qs": "^6.10.5", + "remove-undefined-objects": "^2.0.2" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@readme/openapi-parser": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@readme/openapi-parser/-/openapi-parser-2.5.0.tgz", + "integrity": "sha512-IbymbOqRuUzoIgxfAAR7XJt2FWl6n2yqN09fF5adacGm7W03siA3bj1Emql0X9D2T+RpBYz3x9zDsMhuoMP62A==", + "dependencies": { + "@apidevtools/openapi-schemas": "^2.1.0", + "@apidevtools/swagger-methods": "^3.0.2", + "@jsdevtools/ono": "^7.1.3", + "@readme/better-ajv-errors": "^1.6.0", + "@readme/json-schema-ref-parser": "^1.2.0", + "ajv": "^8.12.0", + "ajv-draft-04": "^1.0.0", + "call-me-maybe": "^1.0.1" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "openapi-types": ">=7" + } + }, + "node_modules/@readme/openapi-parser/node_modules/ajv": { + "version": "8.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz", + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/@readme/openapi-parser/node_modules/ajv-draft-04": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/ajv-draft-04/-/ajv-draft-04-1.0.0.tgz", + "integrity": "sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==", + "peerDependencies": { + "ajv": "^8.5.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/@readme/openapi-parser/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "node_modules/@readme/postman-to-openapi": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@readme/postman-to-openapi/-/postman-to-openapi-4.1.0.tgz", + "integrity": "sha512-VvV2Hzjskz01m8doSn7Ypt6cSZzgjnypVqXy1ipThbyYD6SGiM74VSePXykOODj/43Y2m6zeYedPk/ZLts/HvQ==", + "dependencies": { + "@readme/http-status-codes": "^7.2.0", + "js-yaml": "^4.1.0", + "jsonc-parser": "3.2.0", + "lodash.camelcase": "^4.3.0", + "marked": "^4.3.0", + "mustache": "^4.2.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/@redis/bloom": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", @@ -4795,6 +5050,58 @@ "@temporalio/proto": "1.7.4" } }, + "node_modules/@trpc/client": { + "version": "10.44.1", + "resolved": "https://registry.npmjs.org/@trpc/client/-/client-10.44.1.tgz", + "integrity": "sha512-vTWsykNcgz1LnwePVl2fKZnhvzP9N3GaaLYPkfGINo314ZOS0OBqe9x0ytB2LLUnRVTAAZ2WoONzARd8nHiqrA==", + "funding": [ + "https://trpc.io/sponsor" + ], + "peerDependencies": { + "@trpc/server": "10.44.1" + } + }, + "node_modules/@trpc/server": { + "version": "10.44.1", + "resolved": "https://registry.npmjs.org/@trpc/server/-/server-10.44.1.tgz", + "integrity": "sha512-mF7B+K6LjuboX8I1RZgKE5GA/fJhsJ8tKGK2UBt3Bwik7hepEPb4NJgNr7vO6BK5IYwPdBLRLTctRw6XZx0sRg==", + "funding": [ + "https://trpc.io/sponsor" + ], + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@ts-morph/common": { + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.18.1.tgz", + "integrity": "sha512-RVE+zSRICWRsfrkAw5qCAK+4ZH9kwEFv5h0+/YeHTLieWP7F4wWq4JsKFuNWG+fYh/KF+8rAtgdj5zb2mm+DVA==", + "dependencies": { + "fast-glob": "^3.2.12", + "minimatch": "^5.1.0", + "mkdirp": "^1.0.4", + "path-browserify": "^1.0.1" + } + }, + "node_modules/@ts-morph/common/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/@ts-morph/common/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "license": "MIT" @@ -5027,6 +5334,11 @@ "@types/node": "*" } }, + "node_modules/@types/har-format": { + "version": "1.2.15", + "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.15.tgz", + "integrity": "sha512-RpQH4rXLuvTXKR0zqHq3go0RVXYv/YVqv4TnPH95VbwUxZdQlK1EtcMvQvMpDngHbt13Csh9Z4qT9AbkiQH5BA==" + }, "node_modules/@types/human-to-cron": { "version": "0.3.0", "dev": true, @@ -5910,46 +6222,216 @@ "node": ">= 8" } }, - "node_modules/archiver": { - "version": "5.3.1", - "dev": true, - "license": "MIT", + "node_modules/api": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/api/-/api-6.1.1.tgz", + "integrity": "sha512-we3fnLinpYWlKOHdX4S/Ky9gZvColCnht/qhtv04K2jQbrC/z4SxvZVAT8W8PCC5NLLU4H35r3u4Lt77ZaiY9w==", "dependencies": { - "archiver-utils": "^2.1.0", - "async": "^3.2.3", - "buffer-crc32": "^0.2.1", - "readable-stream": "^3.6.0", - "readdir-glob": "^1.0.0", - "tar-stream": "^2.2.0", - "zip-stream": "^4.1.0" + "@readme/oas-to-har": "^20.0.2", + "@readme/openapi-parser": "^2.4.0", + "caseless": "^0.12.0", + "chalk": "^4.1.2", + "commander": "^10.0.0", + "datauri": "^4.1.0", + "execa": "^5.1.1", + "fetch-har": "^8.1.5", + "figures": "^3.2.0", + "find-cache-dir": "^3.3.1", + "form-data-encoder": "^1.7.2", + "formdata-node": "^4.3.2", + "get-stream": "^6.0.1", + "isomorphic-fetch": "^3.0.0", + "js-yaml": "^4.1.0", + "json-schema-to-ts": "^2.6.2-beta.0", + "json-schema-traverse": "^1.0.0", + "lodash.camelcase": "^4.3.0", + "lodash.deburr": "^4.1.0", + "lodash.merge": "^4.6.2", + "lodash.setwith": "^4.3.2", + "lodash.startcase": "^4.4.0", + "make-dir": "^3.1.0", + "node-abort-controller": "^3.1.1", + "oas": "^20.4.0", + "ora": "^5.4.1", + "prompts": "^2.4.2", + "remove-undefined-objects": "^2.0.2", + "semver": "^7.3.8", + "ssri": "^10.0.1", + "ts-morph": "^17.0.1", + "validate-npm-package-name": "^5.0.0" + }, + "bin": { + "api": "bin/api" }, "engines": { - "node": ">= 10" + "node": ">=16" } }, - "node_modules/archiver-utils": { - "version": "2.1.0", - "dev": true, - "license": "MIT", + "node_modules/api/node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dependencies": { - "glob": "^7.1.4", - "graceful-fs": "^4.2.0", - "lazystream": "^1.0.0", - "lodash.defaults": "^4.2.0", - "lodash.difference": "^4.5.0", - "lodash.flatten": "^4.4.0", - "lodash.isplainobject": "^4.0.6", - "lodash.union": "^4.6.0", - "normalize-path": "^3.0.0", - "readable-stream": "^2.0.0" + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/api/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/api/node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dependencies": { + "restore-cursor": "^3.1.0" }, "engines": { - "node": ">= 6" + "node": ">=8" } }, - "node_modules/archiver-utils/node_modules/isarray": { - "version": "1.0.0", - "dev": true, + "node_modules/api/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "engines": { + "node": ">=14" + } + }, + "node_modules/api/node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/api/node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/api/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, + "node_modules/api/node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/api/node_modules/ora": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", + "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", + "dependencies": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/api/node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/archiver": { + "version": "5.3.1", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.3", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.0.0", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/isarray": { + "version": "1.0.0", + "dev": true, "license": "MIT" }, "node_modules/archiver-utils/node_modules/readable-stream": { @@ -6384,6 +6866,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-me-maybe": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.2.tgz", + "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==" + }, "node_modules/callsites": { "version": "3.1.0", "dev": true, @@ -6417,6 +6904,11 @@ "cargo-cp-artifact": "bin/cargo-cp-artifact.js" } }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" + }, "node_modules/chai": { "version": "4.3.7", "dev": true, @@ -6600,6 +7092,19 @@ "node": ">=0.10.0" } }, + "node_modules/code-block-writer": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-11.0.3.tgz", + "integrity": "sha512-NiujjUFB4SwScJq2bwbYUtXbZhBSlY6vYzm++3Q6oC+U+injTqfPYFK8wS9COOmb2lueqp0ZRB4nK1VYeHgNyw==" + }, + "node_modules/code-error-fragment": { + "version": "0.0.230", + "resolved": "https://registry.npmjs.org/code-error-fragment/-/code-error-fragment-0.0.230.tgz", + "integrity": "sha512-cadkfKp6932H8UkhzE/gcUqhRMNf8jHzkAN7+5Myabswaghu4xABTgPHDCjW+dBAJxj/SpkTYokpzDqY4pCzQw==", + "engines": { + "node": ">= 4" + } + }, "node_modules/color": { "version": "3.2.1", "license": "MIT", @@ -6672,7 +7177,6 @@ }, "node_modules/commondir": { "version": "1.0.1", - "dev": true, "license": "MIT" }, "node_modules/compress-commons": { @@ -6689,6 +7193,27 @@ "node": ">= 10" } }, + "node_modules/compute-gcd": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/compute-gcd/-/compute-gcd-1.2.1.tgz", + "integrity": "sha512-TwMbxBNz0l71+8Sc4czv13h4kEqnchV9igQZBi6QUaz09dnz13juGnnaWWJTRsP3brxOoxeB4SA2WELLw1hCtg==", + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2", + "validate.io-integer-array": "^1.0.0" + } + }, + "node_modules/compute-lcm": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/compute-lcm/-/compute-lcm-1.1.2.tgz", + "integrity": "sha512-OFNPdQAXnQhDSKioX8/XYT6sdUlXwpeMjfd6ApxMJfyZ4GxmLR1xvMERctlYhlHwIiz6CSpBc2+qYKjHGZw4TQ==", + "dependencies": { + "compute-gcd": "^1.2.1", + "validate.io-array": "^1.0.3", + "validate.io-function": "^1.0.2", + "validate.io-integer-array": "^1.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "license": "MIT" @@ -6941,6 +7466,27 @@ "resolved": "https://registry.npmjs.org/crypto-randomuuid/-/crypto-randomuuid-1.0.0.tgz", "integrity": "sha512-/RC5F4l1SCqD/jazwUF6+t34Cd8zTSAGZ7rvvZu1whZUhD2a5MOGKjSGowoGcpj/cbVZk1ZODIooJEQQq3nNAA==" }, + "node_modules/d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dependencies": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + }, + "node_modules/datauri": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/datauri/-/datauri-4.1.0.tgz", + "integrity": "sha512-y17kh32+I82G+ED9MNWFkZiP/Cq/vO1hN9+tSZsT9C9qn3NrvcBnh7crSepg0AQPge1hXx2Ca44s1FRdv0gFWA==", + "dependencies": { + "image-size": "1.0.0", + "mimer": "^2.0.2" + }, + "engines": { + "node": ">= 10" + } + }, "node_modules/date-fns": { "version": "2.29.3", "dev": true, @@ -6973,16 +7519,16 @@ } }, "node_modules/dd-trace": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-4.19.0.tgz", - "integrity": "sha512-fmWqrAlQk1xnKzgPEM8RpQPTalxCa+F8Q/6JQQ1M3AczYrvwmJ45ljw2mB1F4iXeDEj0ycYFH3BRtdywspBLLQ==", + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-4.21.0.tgz", + "integrity": "sha512-nuFHfDJNy039Qns1sbJdIbXeP1ZmUioctwzXfw60k4Dif8dC3RmrHebvW48EbYCKrJj3Awri/vmwcBNf6xNpHw==", "hasInstallScript": true, "dependencies": { - "@datadog/native-appsec": "4.0.0", + "@datadog/native-appsec": "5.0.0", "@datadog/native-iast-rewriter": "2.2.1", "@datadog/native-iast-taint-tracking": "1.6.4", "@datadog/native-metrics": "^2.0.0", - "@datadog/pprof": "4.0.1", + "@datadog/pprof": "4.1.0", "@datadog/sketches-js": "^2.1.0", "@opentelemetry/api": "^1.0.0", "@opentelemetry/core": "^1.14.0", @@ -7008,9 +7554,10 @@ "opentracing": ">=0.12.1", "path-to-regexp": "^0.1.2", "pprof-format": "^2.0.7", - "protobufjs": "^7.2.4", + "protobufjs": "^7.2.5", "retry": "^0.13.1", - "semver": "^7.5.4" + "semver": "^7.5.4", + "tlhunter-sorted-set": "^0.1.0" }, "engines": { "node": ">=16" @@ -7311,10 +7858,54 @@ "version": "1.2.1", "license": "MIT" }, + "node_modules/es5-ext": { + "version": "0.10.62", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", + "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", + "hasInstallScript": true, + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, "node_modules/es6-promise": { "version": "4.2.8", "license": "MIT" }, + "node_modules/es6-symbol": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "dependencies": { + "d": "^1.0.1", + "ext": "^1.1.2" + } + }, + "node_modules/es6-weak-map": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-weak-map/-/es6-weak-map-2.0.3.tgz", + "integrity": "sha512-p5um32HOTO1kP+w7PRnB+5lQ43Z6muuMuIMffvDN8ZB4GcnjLBV6zGStpbASIMk4DCAvEaamhe2zhyCb/QXXsA==", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.46", + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.1" + } + }, "node_modules/esbuild": { "version": "0.17.19", "hasInstallScript": true, @@ -7729,6 +8320,15 @@ "node": ">= 0.6" } }, + "node_modules/event-emitter": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/event-emitter/-/event-emitter-0.3.5.tgz", + "integrity": "sha512-D9rRn9y7kLPnJ+hMq7S/nhvoKwwvVJahBi2BPmx3bvbsEdK3W9ii8cBSGjP+72/LnM4n6fo3+dkCX5FeTQruXA==", + "dependencies": { + "d": "1", + "es5-ext": "~0.10.14" + } + }, "node_modules/event-lite": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/event-lite/-/event-lite-0.1.3.tgz", @@ -7750,7 +8350,6 @@ }, "node_modules/execa": { "version": "5.1.1", - "dev": true, "license": "MIT", "dependencies": { "cross-spawn": "^7.0.3", @@ -7860,6 +8459,19 @@ "version": "2.0.0", "license": "MIT" }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "dependencies": { + "type": "^2.7.2" + } + }, + "node_modules/ext/node_modules/type": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", + "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" + }, "node_modules/external-editor": { "version": "3.1.0", "license": "MIT", @@ -7887,7 +8499,6 @@ }, "node_modules/fast-glob": { "version": "3.2.12", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "^2.0.2", @@ -7902,7 +8513,6 @@ }, "node_modules/fast-glob/node_modules/glob-parent": { "version": "5.1.2", - "dev": true, "license": "ISC", "dependencies": { "is-glob": "^4.0.1" @@ -7920,6 +8530,11 @@ "dev": true, "license": "MIT" }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" + }, "node_modules/fast-xml-parser": { "version": "4.2.5", "funding": [ @@ -7950,7 +8565,6 @@ }, "node_modules/fastq": { "version": "1.13.0", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -7960,6 +8574,22 @@ "version": "4.2.3", "license": "MIT" }, + "node_modules/fetch-har": { + "version": "8.1.5", + "resolved": "https://registry.npmjs.org/fetch-har/-/fetch-har-8.1.5.tgz", + "integrity": "sha512-c9WDro4RWC+suOVRJFNW21cgqTOELRZpvFJgfENvOM7Yt/VA4QeFtRax795SyOpTisdpcl5XNQlQZdAE6HERDA==", + "dependencies": { + "@readme/data-urls": "^1.0.1", + "@types/har-format": "^1.2.8", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">=14" + }, + "optionalDependencies": { + "formdata-node": "^4.3.2" + } + }, "node_modules/figlet": { "version": "1.6.0", "license": "MIT", @@ -8071,7 +8701,6 @@ }, "node_modules/find-cache-dir": { "version": "3.3.2", - "dev": true, "license": "MIT", "dependencies": { "commondir": "^1.0.1", @@ -8175,6 +8804,23 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.9.0.tgz", + "integrity": "sha512-rahaRMkN8P8d/tgK/BLPX+WBVM27NbvdXBxqQujBtkDAIFspaRqN7Od7lfdGQA6KAD+f82fYCLBq1ipvcu8qLw==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, "node_modules/forwarded": { "version": "0.2.0", "license": "MIT", @@ -8297,7 +8943,6 @@ }, "node_modules/get-stream": { "version": "6.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -8382,7 +9027,6 @@ }, "node_modules/grapheme-splitter": { "version": "1.0.4", - "dev": true, "license": "MIT" }, "node_modules/has": { @@ -8450,6 +9094,11 @@ "node": ">= 0.8" } }, + "node_modules/http2-client": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/http2-client/-/http2-client-1.3.5.tgz", + "integrity": "sha512-EC2utToWl4RKfs5zd36Mxq7nzHHBuomZboI0yYL6Y0RmBgT7Sgkq4rQ0ezFTYoIsSs7Tm9SJe+o2FcAg6GBhGA==" + }, "node_modules/https-proxy-agent": { "version": "5.0.1", "license": "MIT", @@ -8463,7 +9112,6 @@ }, "node_modules/human-signals": { "version": "2.1.0", - "dev": true, "license": "Apache-2.0", "engines": { "node": ">=10.17.0" @@ -8538,6 +9186,20 @@ "dev": true, "license": "ISC" }, + "node_modules/image-size": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/image-size/-/image-size-1.0.0.tgz", + "integrity": "sha512-JLJ6OwBfO1KcA+TvJT+v8gbE6iWbj24LyDNFgFEN0lzegn6cC6a/p3NIDaepMsJjQjlUWqIC7wJv8lBFxPNjcw==", + "dependencies": { + "queue": "6.0.2" + }, + "bin": { + "image-size": "bin/image-size.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/import-fresh": { "version": "3.3.0", "dev": true, @@ -8709,6 +9371,11 @@ "node": ">=0.10.0" } }, + "node_modules/is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==" + }, "node_modules/is-stream": { "version": "2.0.1", "license": "MIT", @@ -8745,6 +9412,15 @@ "node": ">=0.10.0" } }, + "node_modules/isomorphic-fetch": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-3.0.0.tgz", + "integrity": "sha512-qvUtwJ3j6qwsF3jLxkZ72qCgjMysPzDfeV240JHiGZsANBYd+EEuu35v7dfrJ9Up0Ak07D7GGSkGhCHTqg/5wA==", + "dependencies": { + "node-fetch": "^2.6.1", + "whatwg-fetch": "^3.4.1" + } + }, "node_modules/istanbul-lib-coverage": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz", @@ -8881,6 +9557,40 @@ "version": "2.3.1", "license": "MIT" }, + "node_modules/json-schema-compare": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz", + "integrity": "sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==", + "dependencies": { + "lodash": "^4.17.4" + } + }, + "node_modules/json-schema-merge-allof": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/json-schema-merge-allof/-/json-schema-merge-allof-0.8.1.tgz", + "integrity": "sha512-CTUKmIlPJbsWfzRRnOXz+0MjIqvnleIXwFTzz+t9T86HnYX/Rozria6ZVGLktAU9e+NygNljveP+yxqtQp/Q4w==", + "dependencies": { + "compute-lcm": "^1.1.2", + "json-schema-compare": "^0.2.2", + "lodash": "^4.17.20" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/json-schema-to-ts": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-2.9.2.tgz", + "integrity": "sha512-h9WqLkTVpBbiaPb5OmeUpz/FBLS/kvIJw4oRCPiEisIu2WjMh+aai0QIY2LoOhRFx5r92taGLcerIrzxKBAP6g==", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@types/json-schema": "^7.0.9", + "ts-algebra": "^1.2.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "license": "MIT" @@ -8890,6 +9600,18 @@ "dev": true, "license": "MIT" }, + "node_modules/json-to-ast": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/json-to-ast/-/json-to-ast-2.1.0.tgz", + "integrity": "sha512-W9Lq347r8tA1DfMvAGn9QNcgYm4Wm7Yc+k8e6vezpMnRT+NHbtlxgNBXRVjXe9YM6eTn6+p/MKOlV/aABJcSnQ==", + "dependencies": { + "code-error-fragment": "0.0.230", + "grapheme-splitter": "^1.0.4" + }, + "engines": { + "node": ">= 4" + } + }, "node_modules/json5": { "version": "2.2.3", "dev": true, @@ -8903,7 +9625,6 @@ }, "node_modules/jsonc-parser": { "version": "3.2.0", - "dev": true, "license": "MIT" }, "node_modules/jsonfile": { @@ -8916,6 +9637,22 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/jsonpointer": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/jsonwebtoken": { "version": "9.0.2", "license": "MIT", @@ -8961,6 +9698,14 @@ "node": ">=0.10.0" } }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "engines": { + "node": ">=6" + } + }, "node_modules/knex": { "version": "2.4.2", "license": "MIT", @@ -9067,6 +9812,14 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "engines": { + "node": ">=6" + } + }, "node_modules/levn": { "version": "0.4.1", "dev": true, @@ -9155,6 +9908,11 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.deburr": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/lodash.deburr/-/lodash.deburr-4.1.0.tgz", + "integrity": "sha512-m/M1U1f3ddMCs6Hq2tAsYThTBDaAKFDX3dwDo97GEYzamXi9SqUpjWi/Rrj/gf3X2n8ktwgZrlP1z6E3v/IExQ==" + }, "node_modules/lodash.defaults": { "version": "4.2.0", "dev": true, @@ -9201,7 +9959,6 @@ }, "node_modules/lodash.merge": { "version": "4.6.2", - "dev": true, "license": "MIT" }, "node_modules/lodash.once": { @@ -9213,10 +9970,20 @@ "resolved": "https://registry.npmjs.org/lodash.pick/-/lodash.pick-4.4.0.tgz", "integrity": "sha512-hXt6Ul/5yWjfklSGvLQl8vM//l3FtyHZeuelpzK6mm99pNvN9yTDruNZPEJZD1oWrqo+izBmB7oUfWgcCX7s4Q==" }, + "node_modules/lodash.setwith": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/lodash.setwith/-/lodash.setwith-4.3.2.tgz", + "integrity": "sha512-Cv2pndcuNDaqDMJ0gbLm5qNG5jyfsL6f8+f5PfZVVNhQCv+y+P5gAKkCdZbtiQlux7nsnWF7UmZd8JEFIo/4tg==" + }, "node_modules/lodash.sortby": { "version": "4.7.0", "license": "MIT" }, + "node_modules/lodash.startcase": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.startcase/-/lodash.startcase-4.4.0.tgz", + "integrity": "sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==" + }, "node_modules/lodash.truncate": { "version": "4.4.2", "dev": true, @@ -9305,6 +10072,14 @@ "node": ">=10" } }, + "node_modules/lru-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/lru-queue/-/lru-queue-0.1.0.tgz", + "integrity": "sha512-BpdYkt9EvGl8OfWHDQPISVpcl5xZthb+XPsbELj5AQXxIC8IriDZIQYjBJPEm5rS420sjZ0TLEzRcq5KdBhYrQ==", + "dependencies": { + "es5-ext": "~0.10.2" + } + }, "node_modules/magic-string": { "version": "0.30.1", "dev": true, @@ -9332,7 +10107,6 @@ }, "node_modules/make-dir": { "version": "3.1.0", - "dev": true, "license": "MIT", "dependencies": { "semver": "^6.0.0" @@ -9346,7 +10120,6 @@ }, "node_modules/make-dir/node_modules/semver": { "version": "6.3.1", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -9356,11 +10129,22 @@ "version": "1.3.6", "license": "ISC" }, - "node_modules/md5": { - "version": "2.3.0", - "license": "BSD-3-Clause", - "dependencies": { - "charenc": "0.0.2", + "node_modules/marked": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-4.3.0.tgz", + "integrity": "sha512-PRsaiG84bK+AMvxziE/lCFss8juXjNaWzVbN5tXAm4XjeaS9NAHhop+PjQxz2A9h8Q4M/xGmzP8vqNwy6JeK0A==", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/md5": { + "version": "2.3.0", + "license": "BSD-3-Clause", + "dependencies": { + "charenc": "0.0.2", "crypt": "0.0.2", "is-buffer": "~1.1.6" } @@ -9382,6 +10166,21 @@ "node": ">= 4.0.0" } }, + "node_modules/memoizee": { + "version": "0.4.15", + "resolved": "https://registry.npmjs.org/memoizee/-/memoizee-0.4.15.tgz", + "integrity": "sha512-UBWmJpLZd5STPm7PMUlOw/TSy972M+z8gcyQ5veOnSDRREz/0bmpyTfKt3/51DhEBqCZQn1udM/5flcSPYhkdQ==", + "dependencies": { + "d": "^1.0.1", + "es5-ext": "^0.10.53", + "es6-weak-map": "^2.0.3", + "event-emitter": "^0.3.5", + "is-promise": "^2.2.2", + "lru-queue": "^0.1.0", + "next-tick": "^1.1.0", + "timers-ext": "^0.1.7" + } + }, "node_modules/merge-descriptors": { "version": "1.0.1", "license": "MIT" @@ -9392,7 +10191,6 @@ }, "node_modules/merge2": { "version": "1.4.1", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -9407,7 +10205,6 @@ }, "node_modules/micromatch": { "version": "4.0.5", - "dev": true, "license": "MIT", "dependencies": { "braces": "^3.0.2", @@ -9444,6 +10241,17 @@ "node": ">= 0.6" } }, + "node_modules/mimer": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mimer/-/mimer-2.0.2.tgz", + "integrity": "sha512-izxvjsB7Ur5HrTbPu6VKTrzxSMBFBqyZQc6dWlZNQ4/wAvf886fD4lrjtFd8IQ8/WmZKdxKjUtqFFNaj3hQ52g==", + "bin": { + "mimer": "bin/mimer" + }, + "engines": { + "node": ">= 12" + } + }, "node_modules/mimic-fn": { "version": "2.1.0", "license": "MIT", @@ -9529,6 +10337,14 @@ "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "license": "ISC" @@ -9590,6 +10406,11 @@ "version": "2.6.2", "license": "MIT" }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" + }, "node_modules/node-abort-controller": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/node-abort-controller/-/node-abort-controller-3.1.1.tgz", @@ -9617,6 +10438,24 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.6.12", "license": "MIT", @@ -9635,6 +10474,17 @@ } } }, + "node_modules/node-fetch-h2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/node-fetch-h2/-/node-fetch-h2-2.3.0.tgz", + "integrity": "sha512-ofRW94Ab0T4AOh5Fk8t0h8OBWrmjb0SSB20xh1H8YnPV9EJ+f5AMoYSUQ2zgJ4Iq2HAK0I2l5/Nequ8YzFS3Hg==", + "dependencies": { + "http2-client": "^1.2.5" + }, + "engines": { + "node": "4.x || >=6.0.0" + } + }, "node_modules/node-gyp-build": { "version": "3.9.0", "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-3.9.0.tgz", @@ -9645,6 +10495,19 @@ "node-gyp-build-test": "build-test.js" } }, + "node_modules/node-readfiles": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/node-readfiles/-/node-readfiles-0.2.0.tgz", + "integrity": "sha512-SU00ZarexNlE4Rjdm83vglt5Y9yiQ+XI1XpflWlb7q7UTN1JUItm69xMeiQCTxtTfnzt+83T8Cx+vI2ED++VDA==", + "dependencies": { + "es6-promise": "^3.2.1" + } + }, + "node_modules/node-readfiles/node_modules/es6-promise": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz", + "integrity": "sha512-SOp9Phqvqn7jtEUxPWdWfWoLmyt2VaJ6MpvP9Comy1MceMXqE6bxvaTu4iaxpYYPzhny28Lc+M87/c2cPK6lDg==" + }, "node_modules/node-releases": { "version": "2.0.10", "license": "MIT" @@ -9758,7 +10621,6 @@ }, "node_modules/npm-run-path": { "version": "4.0.1", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.0.0" @@ -9767,6 +10629,144 @@ "node": ">=8" } }, + "node_modules/oas": { + "version": "20.10.3", + "resolved": "https://registry.npmjs.org/oas/-/oas-20.10.3.tgz", + "integrity": "sha512-dBxDuwn2ssggPMOqEKEzT4sjCqbkol8JozuWrpwD7chcmbKbverj5vpk2kmsczeyguFkLcKUOMcqUUimf9h+IQ==", + "dependencies": { + "@readme/json-schema-ref-parser": "^1.2.0", + "@types/json-schema": "^7.0.11", + "json-schema-merge-allof": "^0.8.1", + "jsonpath-plus": "^7.2.0", + "jsonpointer": "^5.0.0", + "memoizee": "^0.4.14", + "oas-normalize": "^8.4.0", + "openapi-types": "^12.1.1", + "path-to-regexp": "^6.2.0", + "remove-undefined-objects": "^3.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/oas-kit-common": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/oas-kit-common/-/oas-kit-common-1.0.8.tgz", + "integrity": "sha512-pJTS2+T0oGIwgjGpw7sIRU8RQMcUoKCDWFLdBqKB2BNmGpbBMH2sdqAaOXUg8OzonZHU0L7vfJu1mJFEiYDWOQ==", + "dependencies": { + "fast-safe-stringify": "^2.0.7" + } + }, + "node_modules/oas-linter": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/oas-linter/-/oas-linter-3.2.2.tgz", + "integrity": "sha512-KEGjPDVoU5K6swgo9hJVA/qYGlwfbFx+Kg2QB/kd7rzV5N8N5Mg6PlsoCMohVnQmo+pzJap/F610qTodKzecGQ==", + "dependencies": { + "@exodus/schemasafe": "^1.0.0-rc.2", + "should": "^13.2.1", + "yaml": "^1.10.0" + }, + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, + "node_modules/oas-linter/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/oas-normalize": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/oas-normalize/-/oas-normalize-8.4.1.tgz", + "integrity": "sha512-cGODg+AntZteJRHBiYDWKtcO2svWGMXuFWYu2I8b4hOrNiwB3hgDs/ScX3O9mYm6RpLsUIftt6rDHGc8eYG8aA==", + "dependencies": { + "@readme/openapi-parser": "^2.5.0", + "@readme/postman-to-openapi": "^4.1.0", + "js-yaml": "^4.1.0", + "node-fetch": "^2.6.1", + "openapi-types": "^12.1.0", + "swagger2openapi": "^7.0.8" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/oas-resolver": { + "version": "2.5.6", + "resolved": "https://registry.npmjs.org/oas-resolver/-/oas-resolver-2.5.6.tgz", + "integrity": "sha512-Yx5PWQNZomfEhPPOphFbZKi9W93CocQj18NlD2Pa4GWZzdZpSJvYwoiuurRI7m3SpcChrnO08hkuQDL3FGsVFQ==", + "dependencies": { + "node-fetch-h2": "^2.3.0", + "oas-kit-common": "^1.0.8", + "reftools": "^1.1.9", + "yaml": "^1.10.0", + "yargs": "^17.0.1" + }, + "bin": { + "resolve": "resolve.js" + }, + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, + "node_modules/oas-resolver/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/oas-schema-walker": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/oas-schema-walker/-/oas-schema-walker-1.1.5.tgz", + "integrity": "sha512-2yucenq1a9YPmeNExoUa9Qwrt9RFkjqaMAA1X+U7sbb0AqBeTIdMHky9SQQ6iN94bO5NW0W4TRYXerG+BdAvAQ==", + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, + "node_modules/oas-validator": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/oas-validator/-/oas-validator-5.0.8.tgz", + "integrity": "sha512-cu20/HE5N5HKqVygs3dt94eYJfBi0TsZvPVXDhbXQHiEityDN+RROTleefoKRKKJ9dFAF2JBkDHgvWj0sjKGmw==", + "dependencies": { + "call-me-maybe": "^1.0.1", + "oas-kit-common": "^1.0.8", + "oas-linter": "^3.2.2", + "oas-resolver": "^2.5.6", + "oas-schema-walker": "^1.1.5", + "reftools": "^1.1.9", + "should": "^13.2.1", + "yaml": "^1.10.0" + }, + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, + "node_modules/oas-validator/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "engines": { + "node": ">= 6" + } + }, + "node_modules/oas/node_modules/path-to-regexp": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz", + "integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==" + }, + "node_modules/oas/node_modules/remove-undefined-objects": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/remove-undefined-objects/-/remove-undefined-objects-3.0.0.tgz", + "integrity": "sha512-nxG1yYfc/Jxi+bNCBiqKhxVJPE+QvziIOKbD+Dxc93Uisz92v/ZYpo4WR0TJuf+dk2xE8lW2WPJsA3mDFzXy8w==", + "engines": { + "node": ">=16" + } + }, "node_modules/oauth": { "version": "0.10.0", "license": "MIT" @@ -9853,6 +10853,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==" + }, "node_modules/openapi3-ts": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/openapi3-ts/-/openapi3-ts-4.1.2.tgz", @@ -9975,7 +10980,6 @@ }, "node_modules/p-try": { "version": "2.2.0", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -10055,9 +11059,13 @@ "node": ">= 0.4.0" } }, + "node_modules/path-browserify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", + "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + }, "node_modules/path-exists": { "version": "4.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -10221,7 +11229,6 @@ }, "node_modules/pkg-dir": { "version": "4.2.0", - "dev": true, "license": "MIT", "dependencies": { "find-up": "^4.0.0" @@ -10232,7 +11239,6 @@ }, "node_modules/pkg-dir/node_modules/find-up": { "version": "4.1.0", - "dev": true, "license": "MIT", "dependencies": { "locate-path": "^5.0.0", @@ -10244,7 +11250,6 @@ }, "node_modules/pkg-dir/node_modules/locate-path": { "version": "5.0.0", - "dev": true, "license": "MIT", "dependencies": { "p-locate": "^4.1.0" @@ -10255,7 +11260,6 @@ }, "node_modules/pkg-dir/node_modules/p-limit": { "version": "2.3.0", - "dev": true, "license": "MIT", "dependencies": { "p-try": "^2.0.0" @@ -10269,7 +11273,6 @@ }, "node_modules/pkg-dir/node_modules/p-locate": { "version": "4.1.0", - "dev": true, "license": "MIT", "dependencies": { "p-limit": "^2.2.0" @@ -10474,6 +11477,18 @@ "read": "^1.0.4" } }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/proper-lockfile": { "version": "4.1.2", "dev": true, @@ -10580,9 +11595,16 @@ "version": "2.2.0", "license": "MIT" }, + "node_modules/queue": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz", + "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==", + "dependencies": { + "inherits": "~2.0.3" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", - "dev": true, "funding": [ { "type": "github", @@ -10721,6 +11743,14 @@ "@redis/time-series": "1.0.5" } }, + "node_modules/reftools": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/reftools/-/reftools-1.1.9.tgz", + "integrity": "sha512-OVede/NQE13xBQ+ob5CKd5KyeJYU2YInb1bmV4nRoOfquZPkAkxuOXicSe1PvqIuZZ4kD13sPKBbR7UFDmli6w==", + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, "node_modules/regenerate": { "version": "1.4.2", "dev": true, @@ -10739,7 +11769,6 @@ }, "node_modules/regenerator-runtime": { "version": "0.13.11", - "dev": true, "license": "MIT" }, "node_modules/regenerator-transform": { @@ -10795,6 +11824,14 @@ "jsesc": "bin/jsesc" } }, + "node_modules/remove-undefined-objects": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/remove-undefined-objects/-/remove-undefined-objects-2.0.2.tgz", + "integrity": "sha512-b6x4MUtR4YBW1aCoGx3tE4mA2PFjiXSmtSdNmLexQzUdZa4ybnJAItXLKpkcVgCUJIzJtk2DFG402sMSEMlonQ==", + "engines": { + "node": ">=14" + } + }, "node_modules/require-directory": { "version": "2.1.1", "license": "MIT", @@ -10879,7 +11916,6 @@ }, "node_modules/reusify": { "version": "1.0.4", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -10924,7 +11960,6 @@ }, "node_modules/run-parallel": { "version": "1.2.0", - "dev": true, "funding": [ { "type": "github", @@ -11152,6 +12187,54 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/should": { + "version": "13.2.3", + "resolved": "https://registry.npmjs.org/should/-/should-13.2.3.tgz", + "integrity": "sha512-ggLesLtu2xp+ZxI+ysJTmNjh2U0TsC+rQ/pfED9bUZZ4DKefP27D+7YJVVTvKsmjLpIi9jAa7itwDGkDDmt1GQ==", + "dependencies": { + "should-equal": "^2.0.0", + "should-format": "^3.0.3", + "should-type": "^1.4.0", + "should-type-adaptors": "^1.0.1", + "should-util": "^1.0.0" + } + }, + "node_modules/should-equal": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/should-equal/-/should-equal-2.0.0.tgz", + "integrity": "sha512-ZP36TMrK9euEuWQYBig9W55WPC7uo37qzAEmbjHz4gfyuXrEUgF8cUvQVO+w+d3OMfPvSRQJ22lSm8MQJ43LTA==", + "dependencies": { + "should-type": "^1.4.0" + } + }, + "node_modules/should-format": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/should-format/-/should-format-3.0.3.tgz", + "integrity": "sha512-hZ58adtulAk0gKtua7QxevgUaXTTXxIi8t41L3zo9AHvjXO1/7sdLECuHeIN2SRtYXpNkmhoUP2pdeWgricQ+Q==", + "dependencies": { + "should-type": "^1.3.0", + "should-type-adaptors": "^1.0.1" + } + }, + "node_modules/should-type": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/should-type/-/should-type-1.4.0.tgz", + "integrity": "sha512-MdAsTu3n25yDbIe1NeN69G4n6mUnJGtSJHygX3+oN0ZbO3DTiATnf7XnYJdGT42JCXurTb1JI0qOBR65shvhPQ==" + }, + "node_modules/should-type-adaptors": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/should-type-adaptors/-/should-type-adaptors-1.1.0.tgz", + "integrity": "sha512-JA4hdoLnN+kebEp2Vs8eBe9g7uy0zbRo+RMcU0EsNy+R+k049Ki+N5tT5Jagst2g7EAja+euFuoXFCa8vIklfA==", + "dependencies": { + "should-type": "^1.3.0", + "should-util": "^1.0.0" + } + }, + "node_modules/should-util": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/should-util/-/should-util-1.0.1.tgz", + "integrity": "sha512-oXF8tfxx5cDk8r2kYqlkUJzZpDBqVY/II2WhvU0n9Y3XYvAYRmeaf1PvvIvTgPnv4KJ+ES5M0PyDq5Jp+Ygy2g==" + }, "node_modules/side-channel": { "version": "1.0.4", "license": "MIT", @@ -11201,6 +12284,11 @@ "node": ">=10" } }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==" + }, "node_modules/slash": { "version": "3.0.0", "dev": true, @@ -11339,6 +12427,25 @@ "nan": "^2.17.0" } }, + "node_modules/ssri": { + "version": "10.0.5", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-10.0.5.tgz", + "integrity": "sha512-bSf16tAFkGeRlUNDjXu8FzaMQt6g2HZJrun7mtMbIPOddxt3GLMSz5VWUWcqTJUPfLEaDIepGxv+bYQW49596A==", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/ssri/node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "license": "MIT", @@ -11439,7 +12546,6 @@ }, "node_modules/strip-final-newline": { "version": "2.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -11539,6 +12645,40 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/swagger2openapi": { + "version": "7.0.8", + "resolved": "https://registry.npmjs.org/swagger2openapi/-/swagger2openapi-7.0.8.tgz", + "integrity": "sha512-upi/0ZGkYgEcLeGieoz8gT74oWHA0E7JivX7aN9mAf+Tc7BQoRBvnIGHoPDw+f9TXTW4s6kGYCZJtauP6OYp7g==", + "dependencies": { + "call-me-maybe": "^1.0.1", + "node-fetch": "^2.6.1", + "node-fetch-h2": "^2.3.0", + "node-readfiles": "^0.2.0", + "oas-kit-common": "^1.0.8", + "oas-resolver": "^2.5.6", + "oas-schema-walker": "^1.1.5", + "oas-validator": "^5.0.8", + "reftools": "^1.1.9", + "yaml": "^1.10.0", + "yargs": "^17.0.1" + }, + "bin": { + "boast": "boast.js", + "oas-validate": "oas-validate.js", + "swagger2openapi": "swagger2openapi.js" + }, + "funding": { + "url": "https://github.com/Mermade/oas-kit?sponsor=1" + } + }, + "node_modules/swagger2openapi/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "engines": { + "node": ">= 6" + } + }, "node_modules/swc-loader": { "version": "0.2.3", "license": "MIT", @@ -11839,6 +12979,15 @@ "node": ">=8" } }, + "node_modules/timers-ext": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/timers-ext/-/timers-ext-0.1.7.tgz", + "integrity": "sha512-b85NUNzTSdodShTIbky6ZF02e8STtVVfD+fu4aXXShEELpozH+bCpJLYMPZbsABN2wDH7fJpqIoXxJpzbf0NqQ==", + "dependencies": { + "es5-ext": "~0.10.46", + "next-tick": "1" + } + }, "node_modules/tinybench": { "version": "2.5.0", "dev": true, @@ -11860,6 +13009,11 @@ "node": ">=14.0.0" } }, + "node_modules/tlhunter-sorted-set": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz", + "integrity": "sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw==" + }, "node_modules/tmp": { "version": "0.0.33", "license": "MIT", @@ -11921,11 +13075,25 @@ "version": "1.3.0", "license": "MIT" }, + "node_modules/ts-algebra": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz", + "integrity": "sha512-kloPhf1hq3JbCPOTYoOWDKxebWjNb2o/LKnNfkWhxVVisFFmMJPPdJeGoGmM+iRLyoXAR61e08Pb+vUXINg8aA==" + }, "node_modules/ts-interface-checker": { "version": "0.1.13", "dev": true, "license": "Apache-2.0" }, + "node_modules/ts-morph": { + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-17.0.1.tgz", + "integrity": "sha512-10PkHyXmrtsTvZSL+cqtJLTgFXkU43Gd0JCc0Rw6GchWbqKe0Rwgt1v3ouobTZwQzF1mGhDeAlWYBMGRV7y+3g==", + "dependencies": { + "@ts-morph/common": "~0.18.0", + "code-block-writer": "^11.0.3" + } + }, "node_modules/ts-node": { "version": "10.9.1", "license": "MIT", @@ -12127,6 +13295,11 @@ "dev": true, "license": "Unlicense" }, + "node_modules/type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, "node_modules/type-check": { "version": "0.4.0", "dev": true, @@ -12350,6 +13523,38 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/validate.io-array": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/validate.io-array/-/validate.io-array-1.0.6.tgz", + "integrity": "sha512-DeOy7CnPEziggrOO5CZhVKJw6S3Yi7e9e65R1Nl/RTN1vTQKnzjfvks0/8kQ40FP/dsjRAOd4hxmJ7uLa6vxkg==" + }, + "node_modules/validate.io-function": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-function/-/validate.io-function-1.0.2.tgz", + "integrity": "sha512-LlFybRJEriSuBnUhQyG5bwglhh50EpTL2ul23MPIuR1odjO7XaMLFV8vHGwp7AZciFxtYOeiSCT5st+XSPONiQ==" + }, + "node_modules/validate.io-integer": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/validate.io-integer/-/validate.io-integer-1.0.5.tgz", + "integrity": "sha512-22izsYSLojN/P6bppBqhgUDjCkr5RY2jd+N2a3DCAUey8ydvrZ/OkGvFPR7qfOpwR2LC5p4Ngzxz36g5Vgr/hQ==", + "dependencies": { + "validate.io-number": "^1.0.3" + } + }, + "node_modules/validate.io-integer-array": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/validate.io-integer-array/-/validate.io-integer-array-1.0.0.tgz", + "integrity": "sha512-mTrMk/1ytQHtCY0oNO3dztafHYyGU88KL+jRxWuzfOmQb+4qqnWmI+gykvGp8usKZOM0H7keJHEbRaFiYA0VrA==", + "dependencies": { + "validate.io-array": "^1.0.3", + "validate.io-integer": "^1.0.4" + } + }, + "node_modules/validate.io-number": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/validate.io-number/-/validate.io-number-1.0.3.tgz", + "integrity": "sha512-kRAyotcbNaSYoDnXvb4MHg/0a1egJdLwS6oJ38TJY7aw9n93Fl/3blIXdyYvPOp55CNxywooG/3BcrwNrBpcSg==" + }, "node_modules/vary": { "version": "1.1.2", "license": "MIT", @@ -12597,6 +13802,14 @@ "defaults": "^1.0.3" } }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "license": "BSD-2-Clause" @@ -12749,6 +13962,11 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/whatwg-fetch": { + "version": "3.6.20", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz", + "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==" + }, "node_modules/whatwg-url": { "version": "5.0.0", "license": "MIT", @@ -12945,7 +14163,6 @@ }, "node_modules/yargs": { "version": "17.7.1", - "dev": true, "license": "MIT", "dependencies": { "cliui": "^8.0.1", @@ -13108,6 +14325,273 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, + "packages/jobs": { + "name": "@nangohq/nango-jobs", + "version": "0.36.52", + "dependencies": { + "@nangohq/nango-runner": "0.36.52", + "@nangohq/shared": "0.36.52", + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", + "@octokit/rest": "^20.0.1", + "@temporalio/activity": "^1.5.2", + "@temporalio/client": "^1.5.2", + "@temporalio/worker": "^1.5.2", + "@temporalio/workflow": "^1.5.2", + "@trpc/client": "^10.44.1", + "@trpc/server": "^10.44.1", + "@types/fs-extra": "^11.0.1", + "dd-trace": "^4.21.0", + "dotenv": "^16.0.3", + "fs-extra": "^11.1.1", + "js-yaml": "^4.1.0", + "knex": "^2.4.2", + "md5": "^2.3.0", + "nanoid": "3.x", + "uuid": "^9.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.0", + "@types/md5": "^2.3.2", + "@types/node": "^18.7.6", + "@typescript-eslint/eslint-plugin": "^5.0.0", + "@typescript-eslint/parser": "^5.0.0", + "eslint": "^7.32.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-deprecation": "^1.2.1", + "nodemon": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.3.2" + } + }, + "packages/jobs/node_modules/@eslint/eslintrc": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", + "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==", + "dev": true, + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^13.9.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "packages/jobs/node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "packages/jobs/node_modules/@humanwhocodes/config-array": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", + "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "dev": true, + "dependencies": { + "@humanwhocodes/object-schema": "^1.2.0", + "debug": "^4.1.1", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "packages/jobs/node_modules/acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "packages/jobs/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "packages/jobs/node_modules/eslint": { + "version": "7.32.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", + "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==", + "dev": true, + "dependencies": { + "@babel/code-frame": "7.12.11", + "@eslint/eslintrc": "^0.4.3", + "@humanwhocodes/config-array": "^0.5.0", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.1", + "esquery": "^1.4.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.1.2", + "globals": "^13.6.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^6.0.9", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "packages/jobs/node_modules/eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/mysticatea" + } + }, + "packages/jobs/node_modules/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "packages/jobs/node_modules/eslint-visitor-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "packages/jobs/node_modules/eslint/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "packages/jobs/node_modules/espree": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", + "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", + "dev": true, + "dependencies": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.3.1", + "eslint-visitor-keys": "^1.3.0" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "packages/jobs/node_modules/espree/node_modules/eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "packages/jobs/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "packages/jobs/node_modules/ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "packages/jobs/node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "packages/node-client": { "name": "@nangohq/node", "version": "0.36.52", @@ -13122,6 +14606,34 @@ "node": ">=16.7" } }, + "packages/runner": { + "name": "@nangohq/nango-runner", + "version": "0.36.52", + "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", + "dependencies": { + "@nangohq/shared": "0.36.52", + "@trpc/client": "^10.44.0", + "@trpc/server": "^10.44.0", + "api": "^6.1.1" + }, + "devDependencies": { + "@types/node": "^18.7.6", + "typescript": "^5.3.2" + } + }, + "packages/runner/node_modules/typescript": { + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "packages/server": { "name": "@nangohq/nango-server", "version": "0.36.52", diff --git a/package.json b/package.json index de5f7c31777..0a158184eb8 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,9 @@ "packages/frontend", "packages/node-client", "packages/server", - "packages/worker" + "packages/worker", + "packages/runner", + "packages/jobs" ], "scripts": { "create:migration": "cd packages/shared/lib/db && knex migrate:make $1 --esm --knexfile ./knexfile.cjs", @@ -17,7 +19,7 @@ "lint": "eslint . --ext .ts,.tsx", "lint:fix": "eslint . --ext .ts,.tsx --fix", "install:all": "npm i && cd packages/webapp && npm i && cd ../..", - "ts-build": "tsc -b --clean packages/shared packages/server packages/worker packages/cli && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json", + "ts-build": "tsc -b --clean packages/shared packages/server packages/worker packages/cli packages/runner packages/jobs && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json", "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .", "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..", "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..", @@ -30,13 +32,14 @@ "build:prod": "npm run install:all && npm run ts-build && npm run webapp-build:prod", "server:dev:watch": "cd ./packages/server && npm run dev", "worker:dev:watch": "cd ./packages/worker && npm run dev", + "jobs:dev:watch": "npm run dev -w @nangohq/nango-jobs", "webapp:dev:watch": "cd ./packages/webapp && npm run start:hosted", "cli:watch": "cd ./packages/cli && npm run dev", "prepare": "husky install", "build:watch": "tsc -b -w tsconfig.build.json", "dev:watch": "concurrently --kill-others \"npm run build:watch\" \"npm run webapp-build:watch\" \"npm run prettier-watch\" \"npm run shared:watch\" \"npm run cli:watch\"", "watch:dev": "npm run dev:watch", - "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run worker:dev:watch\" ", + "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run worker:dev:watch\" \"npm run jobs:dev:watch\"", "watch:dev:apps": "npm run dev:watch:apps", "dw": "npm run dev:watch", "dwa": "npm run dev:watch:apps", @@ -67,4 +70,4 @@ "@babel/parser": "^7.22.5", "@babel/traverse": "^7.22.5" } -} +} \ No newline at end of file diff --git a/packages/cli/lib/services/local-integration.service.ts b/packages/cli/lib/services/local-integration.service.ts index 3dc4b7e2355..111ccf6df54 100644 --- a/packages/cli/lib/services/local-integration.service.ts +++ b/packages/cli/lib/services/local-integration.service.ts @@ -1,4 +1,4 @@ -import { NangoError, formatScriptError, IntegrationServiceInterface, NangoIntegrationData, NangoSync, localFileService } from '@nangohq/shared'; +import { NangoError, formatScriptError, IntegrationServiceInterface, NangoIntegrationData, NangoSync, NangoProps, localFileService } from '@nangohq/shared'; import * as vm from 'vm'; import * as url from 'url'; import * as crypto from 'crypto'; @@ -9,7 +9,7 @@ class IntegrationService implements IntegrationServiceInterface { syncName: string, _syncId: string, _activityLogId: number | undefined, - nango: NangoSync, + nangoProps: NangoProps, _integrationData: NangoIntegrationData, _environmentId: number, _writeToDb: boolean, @@ -18,6 +18,7 @@ class IntegrationService implements IntegrationServiceInterface { input?: object ): Promise { try { + const nango = new NangoSync(nangoProps); const script: string | null = localFileService.getIntegrationFile(syncName, optionalLoadLocation); if (!script) { diff --git a/packages/jobs/.gitignore b/packages/jobs/.gitignore new file mode 100644 index 00000000000..8c3aba0aad2 --- /dev/null +++ b/packages/jobs/.gitignore @@ -0,0 +1,3 @@ +tsconfig.tsbuildinfo +dist/* +node_modules diff --git a/packages/jobs/Dockerfile b/packages/jobs/Dockerfile new file mode 100644 index 00000000000..ace81c57bd8 --- /dev/null +++ b/packages/jobs/Dockerfile @@ -0,0 +1,20 @@ +FROM node:18-slim + +RUN apt-get update \ + && apt-get install -y ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +ENV SERVER_RUN_MODE=DOCKERIZED + +WORKDIR /nango + +COPY packages/node-client/ packages/node-client/ +COPY packages/shared/ packages/shared/ +COPY packages/jobs/ packages/jobs/ +COPY packages/runner/ packages/runner/ +COPY package*.json ./ + +RUN npm pkg delete scripts.prepare +RUN npm install --omit=dev + +CMD [ "node", "/nango/packages/jobs/dist/app.js" ] diff --git a/packages/jobs/lib/activities.ts b/packages/jobs/lib/activities.ts new file mode 100644 index 00000000000..dda22830920 --- /dev/null +++ b/packages/jobs/lib/activities.ts @@ -0,0 +1,354 @@ +import { Context, CancelledFailure } from '@temporalio/activity'; +import { TimeoutFailure, TerminatedFailure } from '@temporalio/client'; +import { + createSyncJob, + SyncStatus, + SyncType, + Config as ProviderConfig, + configService, + createActivityLog, + LogLevel, + LogActionEnum, + syncRunService, + ServiceResponse, + NangoConnection, + environmentService, + createActivityLogMessage, + createActivityLogAndLogMessage, + ErrorSourceEnum, + errorManager, + metricsManager, + updateSyncJobStatus, + updateLatestJobSyncStatus, + MetricTypes, + isInitialSyncStillRunning, + initialSyncExists, + logger +} from '@nangohq/shared'; +import integrationService from './integration.service.js'; +import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/worker'; + +export async function routeSync(args: InitialSyncArgs): Promise { + const { syncId, syncJobId, syncName, activityLogId, nangoConnection, debug } = args; + let environmentId = nangoConnection?.environment_id; + + // https://typescript.temporal.io/api/classes/activity.Context + const context: Context = Context.current(); + if (!nangoConnection?.environment_id) { + environmentId = (await environmentService.getEnvironmentIdForAccountAssumingProd(nangoConnection.account_id as number)) as number; + } + const syncConfig: ProviderConfig = (await configService.getProviderConfig(nangoConnection?.provider_config_key as string, environmentId)) as ProviderConfig; + + return syncProvider( + syncConfig, + syncId, + syncJobId, + syncName, + SyncType.INITIAL, + { ...nangoConnection, environment_id: environmentId }, + activityLogId, + context, + debug + ); +} + +export async function runAction(args: ActionArgs): Promise { + const { input, nangoConnection, actionName, activityLogId } = args; + + const syncConfig: ProviderConfig = (await configService.getProviderConfig( + nangoConnection?.provider_config_key as string, + nangoConnection?.environment_id as number + )) as ProviderConfig; + + const context: Context = Context.current(); + + const syncRun = new syncRunService({ + integrationService, + writeToDb: true, + nangoConnection, + syncName: actionName, + isAction: true, + syncType: SyncType.ACTION, + activityLogId, + input, + provider: syncConfig.provider, + debug: false, + temporalContext: context + }); + + const actionResults = await syncRun.run(); + + return actionResults; +} + +export async function scheduleAndRouteSync(args: ContinuousSyncArgs): Promise { + const { syncId, activityLogId, syncName, nangoConnection, debug } = args; + let environmentId = nangoConnection?.environment_id; + let syncJobId; + + const initialSyncStillRunning = await isInitialSyncStillRunning(syncId as string); + + if (initialSyncStillRunning) { + const content = `The continuous sync "${syncName}" with sync id ${syncId} did not run because the initial sync is still running. It will attempt to run at the next scheduled time.`; + + logger.log('info', content); + + await metricsManager.capture(MetricTypes.SYNC_OVERLAP, content, LogActionEnum.SYNC, { + environmentId: String(nangoConnection?.environment_id), + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + syncName, + syncId + }); + + return true; + } + + // https://typescript.temporal.io/api/classes/activity.Context + const context: Context = Context.current(); + const syncType = (await initialSyncExists(syncId as string)) ? SyncType.INCREMENTAL : SyncType.INITIAL; + try { + if (!nangoConnection?.environment_id) { + environmentId = (await environmentService.getEnvironmentIdForAccountAssumingProd(nangoConnection.account_id as number)) as number; + syncJobId = await createSyncJob( + syncId as string, + syncType, + SyncStatus.RUNNING, + context.info.workflowExecution.workflowId, + nangoConnection, + context.info.workflowExecution.runId + ); + } else { + syncJobId = await createSyncJob( + syncId as string, + syncType, + SyncStatus.RUNNING, + context.info.workflowExecution.workflowId, + nangoConnection, + context.info.workflowExecution.runId + ); + } + + const syncConfig: ProviderConfig = (await configService.getProviderConfig( + nangoConnection?.provider_config_key as string, + environmentId + )) as ProviderConfig; + + return syncProvider( + syncConfig, + syncId, + syncJobId?.id as number, + syncName, + syncType, + { ...nangoConnection, environment_id: environmentId }, + activityLogId ?? 0, + context, + debug + ); + } catch (err: any) { + const prettyError = JSON.stringify(err, ['message', 'name'], 2); + const log = { + level: 'info' as LogLevel, + success: false, + action: LogActionEnum.SYNC, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: nangoConnection?.connection_id as string, + provider_config_key: nangoConnection?.provider_config_key as string, + provider: '', + session_id: '', + environment_id: environmentId, + operation_name: syncName + }; + const content = `The continuous sync failed to run because of a failure to obtain the provider config for ${syncName} with the following error: ${prettyError}`; + await createActivityLogAndLogMessage(log, { + level: 'error', + environment_id: environmentId, + timestamp: Date.now(), + content + }); + + await metricsManager.capture(MetricTypes.SYNC_FAILURE, content, LogActionEnum.SYNC, { + environmentId: String(environmentId), + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + syncId, + syncName + }); + + await errorManager.report(content, { + environmentId, + source: ErrorSourceEnum.PLATFORM, + operation: LogActionEnum.SYNC, + metadata: { + syncType, + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + syncName + } + }); + + return false; + } +} + +/** + * Sync Provider + * @desc take in a provider, use the nango.yaml config to find + * the integrations where that provider is used and call the sync + * accordingly with the user defined integration code + */ +export async function syncProvider( + syncConfig: ProviderConfig, + syncId: string, + syncJobId: number, + syncName: string, + syncType: SyncType, + nangoConnection: NangoConnection, + existingActivityLogId: number, + temporalContext: Context, + debug = false +): Promise { + try { + let activityLogId = existingActivityLogId; + + if (syncType === SyncType.INCREMENTAL || existingActivityLogId === 0) { + const log = { + level: 'info' as LogLevel, + success: null, + action: LogActionEnum.SYNC, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: nangoConnection?.connection_id as string, + provider_config_key: nangoConnection?.provider_config_key as string, + provider: syncConfig.provider, + session_id: syncJobId ? syncJobId?.toString() : '', + environment_id: nangoConnection?.environment_id as number, + operation_name: syncName + }; + activityLogId = (await createActivityLog(log)) as number; + } + + if (debug) { + await createActivityLogMessage({ + level: 'info', + environment_id: nangoConnection?.environment_id as number, + activity_log_id: activityLogId, + timestamp: Date.now(), + content: `Starting sync ${syncType} for ${syncName} with syncId ${syncId} and syncJobId ${syncJobId} with execution id of ${temporalContext.info.workflowExecution.workflowId} for attempt #${temporalContext.info.attempt}` + }); + } + + const syncRun = new syncRunService({ + integrationService, + writeToDb: true, + syncId, + syncJobId, + nangoConnection, + syncName, + syncType, + activityLogId, + provider: syncConfig.provider, + temporalContext, + debug + }); + + const result = await syncRun.run(); + + return result.response; + } catch (err: any) { + const prettyError = JSON.stringify(err, ['message', 'name'], 2); + const log = { + level: 'info' as LogLevel, + success: false, + action: LogActionEnum.SYNC, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: nangoConnection?.connection_id as string, + provider_config_key: nangoConnection?.provider_config_key as string, + provider: syncConfig.provider, + session_id: syncJobId ? syncJobId?.toString() : '', + environment_id: nangoConnection?.environment_id as number, + operation_name: syncName + }; + const content = `The ${syncType} sync failed to run because of a failure to create the job and run the sync with the error: ${prettyError}`; + + await createActivityLogAndLogMessage(log, { + level: 'error', + environment_id: nangoConnection?.environment_id as number, + timestamp: Date.now(), + content + }); + + await metricsManager.capture(MetricTypes.SYNC_OVERLAP, content, LogActionEnum.SYNC, { + environmentId: String(nangoConnection?.environment_id), + syncId, + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + syncName + }); + + await errorManager.report(content, { + environmentId: nangoConnection?.environment_id as number, + source: ErrorSourceEnum.PLATFORM, + operation: LogActionEnum.SYNC, + metadata: { + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + syncType, + syncName + } + }); + + return false; + } +} + +export async function reportFailure( + error: any, + workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs, + DEFAULT_TIMEOUT: string, + MAXIMUM_ATTEMPTS: number +): Promise { + const { nangoConnection } = workflowArguments; + const type = 'syncName' in workflowArguments ? 'sync' : 'action'; + const name = 'syncName' in workflowArguments ? workflowArguments.syncName : workflowArguments.actionName; + let content = `The ${type} "${name}" failed `; + const context: Context = Context.current(); + + if (error instanceof CancelledFailure) { + content += `due to a cancellation.`; + } else if (error.cause instanceof TerminatedFailure || error.cause.name === 'TerminatedFailure') { + content += `due to a termination.`; + } else if (error.cause instanceof TimeoutFailure || error.cause.name === 'TimeoutFailure') { + if (error.cause.timeoutType === 3) { + content += `due to a timeout with respect to the max schedule length timeout of ${DEFAULT_TIMEOUT}.`; + } else { + content += `due to a timeout and a lack of heartbeat with ${MAXIMUM_ATTEMPTS} attempts.`; + } + } else { + content += `due to a unknown failure.`; + } + + await metricsManager.capture(MetricTypes.FLOW_JOB_TIMEOUT_FAILURE, content, LogActionEnum.SYNC, { + environmentId: String(nangoConnection?.environment_id), + name, + connectionId: nangoConnection?.connection_id as string, + providerConfigKey: nangoConnection?.provider_config_key as string, + error: JSON.stringify(error), + info: JSON.stringify(context.info), + workflowId: context.info.workflowExecution.workflowId, + runId: context.info.workflowExecution.runId + }); + + if (type === 'sync' && 'syncId' in workflowArguments) { + if ('syncJobId' in workflowArguments) { + await updateSyncJobStatus(workflowArguments.syncJobId, SyncStatus.STOPPED); + } else { + await updateLatestJobSyncStatus(workflowArguments.syncId, SyncStatus.STOPPED); + } + } +} diff --git a/packages/jobs/lib/app.ts b/packages/jobs/lib/app.ts new file mode 100644 index 00000000000..eaf88f64d3f --- /dev/null +++ b/packages/jobs/lib/app.ts @@ -0,0 +1,32 @@ +import { Temporal } from './temporal.js'; +import { server } from './server.js'; +import { featureFlags } from '@nangohq/shared'; +import './tracer.js'; + +try { + const port = parseInt(process.env['NANGO_JOBS_PORT'] || '3005', 10); + server.listen(port); + console.log(`🚀 Jobs service ready at http://localhost:${port}`); + + const temporalNs = process.env['TEMPORAL_NAMESPACE'] || 'default'; + const temporal = new Temporal(temporalNs); + + // TODO: remove flag check once jobs is fully enabled by default + let isTemporalStarted = false; + setInterval(async () => { + if (isTemporalStarted) { + return; + } + const isTemporalEnabled = await featureFlags.isEnabled('jobs-temporal', 'global', false); + if (isTemporalEnabled) { + isTemporalStarted = true; + temporal.start(); + } else { + temporal.stop(); + isTemporalStarted = false; + } + }, 5000); +} catch (err) { + console.error(`[JOBS]: ${err}`); + process.exit(1); +} diff --git a/packages/jobs/lib/client.ts b/packages/jobs/lib/client.ts new file mode 100644 index 00000000000..e875ed6f6a6 --- /dev/null +++ b/packages/jobs/lib/client.ts @@ -0,0 +1,8 @@ +import { CreateTRPCProxyClient, createTRPCProxyClient, httpBatchLink } from '@trpc/client'; +import type { AppRouter } from './server.js'; + +export function getJobsClient(url: string): CreateTRPCProxyClient { + return createTRPCProxyClient({ + links: [httpBatchLink({ url })] + }); +} diff --git a/packages/jobs/lib/integration.service.ts b/packages/jobs/lib/integration.service.ts new file mode 100644 index 00000000000..4c40b284679 --- /dev/null +++ b/packages/jobs/lib/integration.service.ts @@ -0,0 +1,149 @@ +import type { Context } from '@temporalio/activity'; +import { + IntegrationServiceInterface, + createActivityLogMessage, + NangoIntegrationData, + NangoProps, + localFileService, + remoteFileService, + isCloud, + getEnv, + ServiceResponse, + NangoError, + formatScriptError, + featureFlags +} from '@nangohq/shared'; +import { getRunner } from './runner/runner.js'; +import tracer from './tracer.js'; + +class IntegrationService implements IntegrationServiceInterface { + public runningScripts: { [key: string]: Context } = {}; + + constructor() { + this.sendHeartbeat(); + } + + async runScript( + syncName: string, + syncId: string, + activityLogId: number | undefined, + nangoProps: NangoProps, + integrationData: NangoIntegrationData, + environmentId: number, + writeToDb: boolean, + isAction: boolean, + optionalLoadLocation?: string, + input?: object, + temporalContext?: Context + ): Promise> { + const span = tracer + .startSpan('runScript') + .setTag('accountId', nangoProps.accountId) + .setTag('environmentId', nangoProps.environmentId) + .setTag('connectionId', nangoProps.connectionId) + .setTag('providerConfigKey', nangoProps.providerConfigKey) + .setTag('syncId', nangoProps.syncId) + .setTag('syncName', syncName); + try { + const script: string | null = + isCloud() && !optionalLoadLocation + ? await remoteFileService.getFile(integrationData.fileLocation as string, environmentId) + : localFileService.getIntegrationFile(syncName, optionalLoadLocation); + + if (!script) { + const content = `Unable to find integration file for ${syncName}`; + + if (activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content, + timestamp: Date.now() + }); + } + } + + if (!script && activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content: `Unable to find integration file for ${syncName}`, + timestamp: Date.now() + }); + + const error = new NangoError('Unable to find integration file', 404); + + return { success: false, error, response: null }; + } + + if (temporalContext) { + this.runningScripts[syncId] = temporalContext; + } + if (nangoProps.accountId == null) { + throw new Error(`No accountId provided (instead ${nangoProps.accountId})`); + } + + const accountId = nangoProps.accountId; + const isRunnerForAccountEnabled = await featureFlags.isEnabled('runner-for-account', `${accountId}`, false); + const runnerSuffix = isRunnerForAccountEnabled ? `${accountId}` : 'default'; + const runnerId = `${getEnv()}-runner-account-${runnerSuffix}`; + const runner = await getRunner(runnerId); + + const runSpan = tracer.startSpan('runner.run', { childOf: span }).setTag('runnerId', runnerId); + try { + // TODO: request sent to the runner for it to run the script is synchronous. + // TODO: Make the request return immediately and have the runner ping the job service when it's done. + const res = await runner.client.run.mutate({ nangoProps, code: script as string, codeParams: input as object, isAction }); + return { success: true, error: null, response: res }; + } catch (err: any) { + const errorType = isAction ? 'action_script_failure' : 'sync_script_failure'; + const { success, error, response } = formatScriptError(err, errorType, syncName); + + if (activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content: error.message, + timestamp: Date.now() + }); + } + return { success, error, response }; + } finally { + runSpan.finish(); + } + } catch (err) { + const errorMessage = JSON.stringify(err, ['message', 'name', 'stack'], 2); + const content = `There was an error running integration '${syncName}': ${errorMessage}`; + + if (activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content, + timestamp: Date.now() + }); + } + + return { success: false, error: new NangoError(content, 500), response: null }; + } finally { + delete this.runningScripts[syncId]; + span.finish(); + } + } + + private sendHeartbeat() { + setInterval(() => { + Object.keys(this.runningScripts).forEach((syncId) => { + const context = this.runningScripts[syncId]; + + context?.heartbeat(); + }); + }, 300000); + } +} + +export default new IntegrationService(); diff --git a/packages/jobs/lib/models/worker.ts b/packages/jobs/lib/models/worker.ts new file mode 100644 index 00000000000..fafa9297ec9 --- /dev/null +++ b/packages/jobs/lib/models/worker.ts @@ -0,0 +1,26 @@ +import type { NangoConnection, NangoIntegrationData } from '@nangohq/shared'; + +export interface InitialSyncArgs { + syncId: string; + syncJobId: number; + syncName: string; + activityLogId: number; + nangoConnection: NangoConnection; + debug?: boolean; +} + +export interface ContinuousSyncArgs { + syncId: string; + activityLogId: number; + syncName: string; + syncData: NangoIntegrationData; + nangoConnection: NangoConnection; + debug?: boolean; +} + +export interface ActionArgs { + input: object; + actionName: string; + nangoConnection: NangoConnection; + activityLogId: number; +} diff --git a/packages/jobs/lib/runner/local.runner.ts b/packages/jobs/lib/runner/local.runner.ts new file mode 100644 index 00000000000..01c9224a9bb --- /dev/null +++ b/packages/jobs/lib/runner/local.runner.ts @@ -0,0 +1,55 @@ +import type { Runner } from './runner.js'; +import { execSync, spawn, ChildProcess } from 'child_process'; +import { getRunnerClient } from '@nangohq/nango-runner'; + +export class LocalRunner implements Runner { + constructor(public readonly client: any, private readonly childProcess: ChildProcess) {} + + async stop(): Promise { + this.childProcess.kill(); + } + + static async get(runnerId: string): Promise { + try { + const port = Math.floor(Math.random() * 1000) + 11000; // random port between 11000 and 12000; + let nodePath = ''; + try { + nodePath = execSync('which node', { encoding: 'utf-8' }).trim(); + } catch (err) { + throw new Error('Unable to find node'); + } + + const nangoRunnerPath = process.env['NANGO_RUNNER_PATH'] || '../runner/dist/app.js'; + + const cmd = nodePath; + const runnerLocation = `${nangoRunnerPath}`; + const cmdOptions = [runnerLocation, port.toString(), runnerId]; + console.log(`[Runner] Starting runner with command: ${cmd} ${cmdOptions.join(' ')} `); + + const childProcess = spawn(cmd, cmdOptions, { + stdio: [null, null, null] + }); + + if (!childProcess) { + throw new Error('Unable to spawn runner process'); + } + + if (childProcess.stdout) { + childProcess.stdout.on('data', (data) => { + console.log(`[Runner] ${data.toString()} `); + }); + } + + if (childProcess.stderr) { + childProcess.stderr.on('data', (data) => { + console.log(`[Runner][ERROR] ${data.toString()} `); + }); + } + + const client = getRunnerClient(`http://localhost:${port}`); + return new LocalRunner(client, childProcess); + } catch (err) { + throw new Error(`Unable to get runner ${runnerId}: ${err}`); + } + } +} diff --git a/packages/jobs/lib/runner/render.runner.ts b/packages/jobs/lib/runner/render.runner.ts new file mode 100644 index 00000000000..042bfa4df36 --- /dev/null +++ b/packages/jobs/lib/runner/render.runner.ts @@ -0,0 +1,62 @@ +import type { Runner } from './runner.js'; +import { getRunnerClient } from '@nangohq/nango-runner'; +import { getEnv } from '@nangohq/shared'; +import api from 'api'; + +const render = api('@render-api/v1.0#aiie8wizhlp1is9bu'); +render.auth(process.env['RENDER_API_KEY']); + +export class RenderRunner implements Runner { + constructor(public readonly client: any, private readonly serviceId: string) {} + + async stop(): Promise { + render.suspendService({ serviceId: this.serviceId }); + } + + static async get(runnerId: string): Promise { + try { + let svc = null; + // check if runner exists, if not, create it + let res = await render.getServices({ name: runnerId, type: 'private_service', limit: '1' }); + if (res.data.length > 0) { + svc = res.data[0].service; + } else { + const imageTag = getEnv(); + const ownerId = process.env['RUNNER_OWNER_ID']; + if (!ownerId) { + throw new Error('RUNNER_OWNER_ID is not set'); + } + res = await render.createService({ + type: 'private_service', + name: runnerId, + ownerId: ownerId, + image: { ownerId: ownerId, imagePath: `nangohq/nango-runner:${imageTag}` }, + serviceDetails: { env: 'image' }, + envVars: [ + { key: 'NODE_ENV', value: process.env['NODE_ENV'] }, + { key: 'NANGO_CLOUD', value: process.env['NANGO_CLOUD'] }, + { key: 'NANGO_DB_HOST', value: process.env['NANGO_DB_HOST'] }, + { key: 'NANGO_DB_NAME', value: process.env['NANGO_DB_NAME'] }, + { key: 'NANGO_DB_PASSWORD', value: process.env['NANGO_DB_PASSWORD'] }, + { key: 'NANGO_DB_PORT', value: process.env['NANGO_DB_PORT'] }, + { key: 'NANGO_DB_SSL', value: process.env['NANGO_DB_SSL'] }, + { key: 'NANGO_ENCRYPTION_KEY', value: process.env['NANGO_ENCRYPTION_KEY'] } + ] + }); + svc = res.data.service; + } + if (!svc) { + throw new Error(`Unable to create runner instance ${runnerId}`); + } + // check if runner is suspended, if so, resume it + if (svc.suspended === 'suspended') { + const res = await render.resumeService({ serviceId: svc.id }); + console.log(res); + } + const client = getRunnerClient(`http://${runnerId}`); + return new RenderRunner(client, svc.id); + } catch (err) { + throw new Error(`Unable to get runner ${runnerId}: ${err}`); + } + } +} diff --git a/packages/jobs/lib/runner/runner.ts b/packages/jobs/lib/runner/runner.ts new file mode 100644 index 00000000000..1f9ccb06dfb --- /dev/null +++ b/packages/jobs/lib/runner/runner.ts @@ -0,0 +1,29 @@ +import { LocalRunner } from './local.runner.js'; +import { RenderRunner } from './render.runner.js'; + +export async function getRunner(runnerId: string): Promise { + const isRender = process.env['IS_RENDER'] === 'true'; + const runner = isRender ? await RenderRunner.get(runnerId) : await LocalRunner.get(runnerId); + + // Wait for runner to start and be healthy + const timeoutMs = isRender ? 90000 : 10000; + let healthCheck = false; + let startTime = Date.now(); + while (!healthCheck && Date.now() - startTime < timeoutMs) { + try { + await runner.client.health.query(); + healthCheck = true; + } catch (err) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + } + if (!healthCheck) { + throw new Error(`Runner '${runnerId}' hasn't started after ${timeoutMs}ms,`); + } + return runner; +} + +export interface Runner { + client: any; + stop(): Promise; +} diff --git a/packages/jobs/lib/server.ts b/packages/jobs/lib/server.ts new file mode 100644 index 00000000000..91671409113 --- /dev/null +++ b/packages/jobs/lib/server.ts @@ -0,0 +1,24 @@ +import { initTRPC } from '@trpc/server'; +import { createHTTPServer } from '@trpc/server/adapters/standalone'; + +const t = initTRPC.create(); + +const router = t.router; +const publicProcedure = t.procedure; +// TODO: add logging middleware + +const appRouter = router({ + health: healthProcedure() +}); + +export type AppRouter = typeof appRouter; + +export const server = createHTTPServer({ + router: appRouter +}); + +function healthProcedure() { + return publicProcedure.query(async () => { + return { status: 'ok' }; + }); +} diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts new file mode 100644 index 00000000000..8bf51ad0535 --- /dev/null +++ b/packages/jobs/lib/temporal.ts @@ -0,0 +1,66 @@ +import { Worker, NativeConnection } from '@temporalio/worker'; +import fs from 'fs-extra'; +import * as dotenv from 'dotenv'; +import { createRequire } from 'module'; +import * as activities from './activities.js'; +import { TASK_QUEUE, isProd } from '@nangohq/shared'; + +export class Temporal { + namespace: string; + worker: Worker | null; + + constructor(namespace: string) { + this.namespace = namespace; + this.worker = null; + } + + async start() { + console.log('Starting Temporal worker'); + + if (process.env['SERVER_RUN_MODE'] !== 'DOCKERIZED') { + dotenv.config({ path: '../../.env' }); + } + + let crt: Buffer | null = null; + let key: Buffer | null = null; + + if (isProd()) { + crt = await fs.readFile(`/etc/secrets/${this.namespace}.crt`); + key = await fs.readFile(`/etc/secrets/${this.namespace}.key`); + } + + const connection = await NativeConnection.connect({ + address: process.env['TEMPORAL_ADDRESS'] || 'localhost:7233', + tls: !isProd() + ? false + : { + clientCertPair: { + crt: crt as Buffer, + key: key as Buffer + } + } + }); + + this.worker = await Worker.create({ + connection, + namespace: this.namespace, + workflowsPath: createRequire(import.meta.url).resolve('./workflows'), + activities, + taskQueue: TASK_QUEUE + }); + // Worker connects to localhost by default and uses console.error for logging. + // Customize the Worker by passing more options to create(): + // https://typescript.temporal.io/api/classes/worker.Worker + // If you need to configure server connection parameters, see docs: + // https://docs.temporal.io/typescript/security#encryption-in-transit-with-mtls + + await this.worker.run(); + } + + stop() { + if (this.worker) { + console.log('Stopping Temporal worker'); + this.worker.shutdown(); + } + } +} diff --git a/packages/jobs/lib/tracer.ts b/packages/jobs/lib/tracer.ts new file mode 100644 index 00000000000..78f78b913c1 --- /dev/null +++ b/packages/jobs/lib/tracer.ts @@ -0,0 +1,9 @@ +import tracer from 'dd-trace'; +import { isCloud } from '@nangohq/shared'; +if (isCloud()) { + tracer.init({ + service: 'nango-jobs' + }); +} + +export default tracer; diff --git a/packages/jobs/lib/workflows.ts b/packages/jobs/lib/workflows.ts new file mode 100644 index 00000000000..58e7cbaf3f7 --- /dev/null +++ b/packages/jobs/lib/workflows.ts @@ -0,0 +1,46 @@ +import { proxyActivities } from '@temporalio/workflow'; +import type * as activities from './activities.js'; +import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/worker'; + +const DEFAULT_TIMEOUT = '24 hours'; +const MAXIMUM_ATTEMPTS = 3; + +const { reportFailure, routeSync, scheduleAndRouteSync, runAction } = proxyActivities({ + startToCloseTimeout: DEFAULT_TIMEOUT, + scheduleToCloseTimeout: DEFAULT_TIMEOUT, + retry: { + initialInterval: '5m', + maximumAttempts: MAXIMUM_ATTEMPTS + }, + heartbeatTimeout: '30m' +}); + +export async function initialSync(args: InitialSyncArgs): Promise { + try { + return await routeSync(args); + } catch (e: any) { + await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); + + return false; + } +} + +export async function continuousSync(args: ContinuousSyncArgs): Promise { + try { + return await scheduleAndRouteSync(args); + } catch (e: any) { + await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); + + return false; + } +} + +export async function action(args: ActionArgs): Promise { + try { + return await runAction(args); + } catch (e: any) { + await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); + + return { success: false }; + } +} diff --git a/packages/jobs/nodemon.json b/packages/jobs/nodemon.json new file mode 100644 index 00000000000..cc4898c4d74 --- /dev/null +++ b/packages/jobs/nodemon.json @@ -0,0 +1,7 @@ +{ + "watch": ["lib", "../shared/lib", "../../.env"], + "ext": "ts,json", + "ignore": ["lib/**/*.spec.ts"], + "exec": "ts-node-esm -r dotenv/config lib/app.ts dotenv_config_path=./../../.env", + "signal": "SIGTERM" +} \ No newline at end of file diff --git a/packages/jobs/package.json b/packages/jobs/package.json new file mode 100644 index 00000000000..ff9e92cd5c3 --- /dev/null +++ b/packages/jobs/package.json @@ -0,0 +1,52 @@ +{ + "name": "@nangohq/nango-jobs", + "version": "0.36.52", + "type": "module", + "main": "dist/app.js", + "scripts": { + "start": "tsc && node dist/app.js", + "dev": "nodemon", + "build": "rimraf dist && tsc" + }, + "keywords": [], + "repository": { + "type": "git", + "url": "https://github.com/NangoHQ/nango", + "directory": "packages/jobs" + }, + "dependencies": { + "@nangohq/nango-runner": "0.36.52", + "@nangohq/shared": "0.36.52", + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", + "@octokit/rest": "^20.0.1", + "@temporalio/activity": "^1.5.2", + "@temporalio/client": "^1.5.2", + "@temporalio/worker": "^1.5.2", + "@temporalio/workflow": "^1.5.2", + "@trpc/client": "^10.44.1", + "@trpc/server": "^10.44.1", + "@types/fs-extra": "^11.0.1", + "dd-trace": "^4.21.0", + "dotenv": "^16.0.3", + "fs-extra": "^11.1.1", + "js-yaml": "^4.1.0", + "knex": "^2.4.2", + "md5": "^2.3.0", + "nanoid": "3.x", + "uuid": "^9.0.0" + }, + "devDependencies": { + "@tsconfig/node16": "^1.0.0", + "@types/md5": "^2.3.2", + "@types/node": "^18.7.6", + "@typescript-eslint/eslint-plugin": "^5.0.0", + "@typescript-eslint/parser": "^5.0.0", + "eslint": "^7.32.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-deprecation": "^1.2.1", + "nodemon": "^3.0.1", + "ts-node": "^10.9.1", + "typescript": "^5.3.2" + } +} diff --git a/packages/jobs/tsconfig.json b/packages/jobs/tsconfig.json new file mode 100644 index 00000000000..dd2652e614c --- /dev/null +++ b/packages/jobs/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "lib", + "outDir": "dist" + }, + "references": [{ "path": "../shared" }], + "include": ["lib/**/*"] +} \ No newline at end of file diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile new file mode 100644 index 00000000000..be6335fafa7 --- /dev/null +++ b/packages/runner/Dockerfile @@ -0,0 +1,19 @@ +FROM node:18-slim + +RUN apt-get update \ + && apt-get install -y ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +ENV SERVER_RUN_MODE=DOCKERIZED + +WORKDIR /nango + +COPY packages/node-client/ packages/node-client/ +COPY packages/shared/ packages/shared/ +COPY packages/runner/ packages/runner/ +COPY package*.json ./ + +RUN npm pkg delete scripts.prepare +RUN npm install --omit=dev + +CMD [ "node", "/nango/packages/runner/dist/app.js", "80", "dockerized-runner" ] diff --git a/packages/runner/lib/app.ts b/packages/runner/lib/app.ts new file mode 100644 index 00000000000..ff03b5b20d4 --- /dev/null +++ b/packages/runner/lib/app.ts @@ -0,0 +1,11 @@ +import { server } from './server.js'; + +try { + const port = parseInt(process.argv[2] || '3006', 10); + const id = process.argv[3] || 'unknown-id'; + server.listen(port); + console.log(`🚀 Runner '${id}' ready at http://localhost:${port}`); +} catch (err) { + console.error(`Unable to start runner: ${err}`); + process.exit(1); +} diff --git a/packages/runner/lib/client.ts b/packages/runner/lib/client.ts new file mode 100644 index 00000000000..569d6a8d7a7 --- /dev/null +++ b/packages/runner/lib/client.ts @@ -0,0 +1,8 @@ +import { CreateTRPCProxyClient, createTRPCProxyClient, httpBatchLink } from '@trpc/client'; +import type { AppRouter } from './server.js'; + +export function getRunnerClient(url: string): CreateTRPCProxyClient { + return createTRPCProxyClient({ + links: [httpBatchLink({ url })] + }); +} diff --git a/packages/runner/lib/client.unit.test.ts b/packages/runner/lib/client.unit.test.ts new file mode 100644 index 00000000000..a9c85460451 --- /dev/null +++ b/packages/runner/lib/client.unit.test.ts @@ -0,0 +1,43 @@ +import { expect, describe, it, beforeAll } from 'vitest'; +import { getRunnerClient } from './client.js'; +import { server } from './server.js'; + +describe('Runner client', () => { + const port = 3095; + const serverUrl = `http://localhost:${port}`; + let client: ReturnType; + + beforeAll(() => { + client = getRunnerClient(serverUrl); + server.listen(port); + }); + + it('should get server health', async () => { + const result = await client.health.query(); + expect(result).toEqual({ status: 'ok' }); + }); + + it('should run code', async () => { + const nangoProps = { + host: 'http://localhost:3003', + connectionId: 'connection-id', + environmentId: 1, + providerConfigKey: 'provider-config-key', + activityLogId: 1, + secretKey: 'secret-key', + nangoConnectionId: 1, + syncId: 'sync-id', + syncJobId: 1, + lastSyncDate: new Date(), + dryRun: true, + attributes: {}, + track_deletes: false, + logMessages: [], + stubbedMetadata: {} + }; + const jsCode = 'f = () => { return [1, 2, 3] }; exports.default = f'; + const isAction = true; + const run = client.run.mutate({ nangoProps, isAction, code: jsCode }); + await expect(run).resolves.toEqual([1, 2, 3]); + }); +}); diff --git a/packages/runner/lib/exec.ts b/packages/runner/lib/exec.ts new file mode 100644 index 00000000000..add75a6a6b0 --- /dev/null +++ b/packages/runner/lib/exec.ts @@ -0,0 +1,46 @@ +import type { NangoProps } from '@nangohq/shared'; +import { NangoSync } from '@nangohq/shared'; +import * as vm from 'vm'; +import * as url from 'url'; +import * as crypto from 'crypto'; + +export async function exec(nangoProps: NangoProps, isAction: boolean, code: string, codeParams?: object): Promise { + const nangoSync = new NangoSync(nangoProps); + const wrappedCode = ` + (function() { + var module = { exports: {} }; + var exports = module.exports; + ${code} + return module.exports.default || module.exports; + })(); + `; + + try { + const script = new vm.Script(wrappedCode); + const sandbox = { + console, + require: (moduleName: string) => { + switch (moduleName) { + case 'url': + return url; + case 'crypto': + return crypto; + default: + throw new Error(`Module '${moduleName}' is not allowed`); + } + } + }; + const context = vm.createContext(sandbox); + const f = script.runInContext(context); + if (!f || typeof f !== 'function') { + throw new Error(`Default exports is not a function but a ${typeof f}`); + } + if (isAction) { + return await f(nangoSync, codeParams); + } else { + return await f(nangoSync); + } + } catch (error) { + throw new Error(`Error executing code '${error}'`); + } +} diff --git a/packages/runner/lib/index.ts b/packages/runner/lib/index.ts new file mode 100644 index 00000000000..9b3f4f9fb54 --- /dev/null +++ b/packages/runner/lib/index.ts @@ -0,0 +1 @@ +export { getRunnerClient } from './client.js'; diff --git a/packages/runner/lib/server.ts b/packages/runner/lib/server.ts new file mode 100644 index 00000000000..6330a41e878 --- /dev/null +++ b/packages/runner/lib/server.ts @@ -0,0 +1,49 @@ +import { initTRPC } from '@trpc/server'; +import { createHTTPServer } from '@trpc/server/adapters/standalone'; +import type { NangoProps } from '@nangohq/shared'; +import { exec } from './exec.js'; + +const t = initTRPC.create(); + +// const logging = t.middleware(async (opts) => { +// // TODO +// console.log(`[Runner] Received: ${JSON.stringify(opts)}`); +// const result = await opts.next(); +// return result; +// }); + +const router = t.router; +const publicProcedure = t.procedure; //.use(logging); + +interface RunParams { + nangoProps: NangoProps; + isAction: boolean; + code: string; + codeParams?: object; +} + +const appRouter = router({ + health: healthProcedure(), + run: runProcedure() +}); + +export type AppRouter = typeof appRouter; + +export const server = createHTTPServer({ + router: appRouter +}); + +function healthProcedure() { + return publicProcedure.query(async () => { + return { status: 'ok' }; + }); +} + +function runProcedure() { + return publicProcedure + .input((input) => input as RunParams) + .mutation(async ({ input }) => { + const { nangoProps, code, codeParams } = input; + return await exec(nangoProps, input.isAction, code, codeParams); + }); +} diff --git a/packages/runner/nodemon.json b/packages/runner/nodemon.json new file mode 100644 index 00000000000..2c50525bc9d --- /dev/null +++ b/packages/runner/nodemon.json @@ -0,0 +1,6 @@ +{ + "watch": ["lib", "../shared/lib", "../../.env"], + "ext": "ts,json", + "ignore": ["lib/**/*.test.ts"], + "exec": "tsc && ts-node-esm -r dotenv/config lib/server.ts dotenv_config_path=./../../.env" +} \ No newline at end of file diff --git a/packages/runner/package.json b/packages/runner/package.json new file mode 100644 index 00000000000..611d1c16076 --- /dev/null +++ b/packages/runner/package.json @@ -0,0 +1,30 @@ +{ + "name": "@nangohq/nango-runner", + "version": "0.36.52", + "description": "Runs Nango integration code", + "type": "module", + "main": "dist/index.js", + "typings": "dist/index.d.ts", + "scripts": { + "build": "rimraf ./dist && tsc", + "start": "node dist/app.js", + "dev": "nodemon" + }, + "keywords": [], + "repository": { + "type": "git", + "url": "https://github.com/NangoHQ/nango", + "directory": "packages/runner" + }, + "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", + "dependencies": { + "@nangohq/shared": "0.36.52", + "@trpc/client": "^10.44.0", + "@trpc/server": "^10.44.0", + "api": "^6.1.1" + }, + "devDependencies": { + "@types/node": "^18.7.6", + "typescript": "^5.3.2" + } +} \ No newline at end of file diff --git a/packages/runner/tsconfig.json b/packages/runner/tsconfig.json new file mode 100644 index 00000000000..dd2652e614c --- /dev/null +++ b/packages/runner/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "lib", + "outDir": "dist" + }, + "references": [{ "path": "../shared" }], + "include": ["lib/**/*"] +} \ No newline at end of file diff --git a/packages/shared/lib/models/Sync.ts b/packages/shared/lib/models/Sync.ts index 203ea9ab43b..6158a37bda0 100644 --- a/packages/shared/lib/models/Sync.ts +++ b/packages/shared/lib/models/Sync.ts @@ -1,7 +1,7 @@ import type { Context } from '@temporalio/activity'; import { LogActionEnum } from './Activity.js'; import type { HTTP_VERB, Timestamps, TimestampsAndDeleted } from './Generic.js'; -import type { NangoSync } from '../sdk/sync.js'; +import type { NangoProps } from '../sdk/sync.js'; import type { NangoIntegrationData, NangoSyncEndpoint } from './NangoConfig.js'; export enum SyncStatus { @@ -293,7 +293,7 @@ export interface IntegrationServiceInterface { syncName: string, syncId: string, activityLogId: number | undefined, - nango: NangoSync, + nangoProps: NangoProps, integrationData: NangoIntegrationData, environmentId: number, writeToDb: boolean, diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index b4e0c637d59..f35371cbf5b 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -170,9 +170,10 @@ interface Connection { credentials: AuthCredentials; } -interface NangoProps { +export interface NangoProps { host?: string; secretKey: string; + accountId?: number; connectionId?: string; environmentId?: number; activityLogId?: number; diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 47ebfb6c7fb..421161a6282 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -14,7 +14,6 @@ import { getDeletedKeys, takeSnapshot, clearOldRecords, syncUpdateAtForDeletedRe import environmentService from '../environment.service.js'; import slackNotificationService from './notification/slack.service.js'; import webhookService from './notification/webhook.service.js'; -import { NangoSync } from '../../sdk/sync.js'; import { isCloud, getApiUrl, JAVASCRIPT_PRIMITIVES } from '../../utils/utils.js'; import errorManager, { ErrorSourceEnum } from '../../utils/error.manager.js'; import { NangoError } from '../../utils/error.js'; @@ -258,8 +257,9 @@ export default class SyncRun { } } - const nango = new NangoSync({ + const nangoProps = { host: optionalHost || getApiUrl(), + accountId: environment?.account_id as number, connectionId: String(this.nangoConnection?.connection_id), environmentId: this.nangoConnection?.environment_id as number, providerConfigKey: String(this.nangoConnection?.provider_config_key), @@ -274,7 +274,7 @@ export default class SyncRun { track_deletes: trackDeletes as boolean, logMessages: this.logMessages, stubbedMetadata: this.stubbedMetadata - }); + }; if (this.debug) { const content = `Last sync date is ${lastSyncDate}`; @@ -306,7 +306,7 @@ export default class SyncRun { (this.syncId as string) || `${this.syncName}-${this.nangoConnection.environment_id}-${this.nangoConnection.provider_config_key}-${this.nangoConnection.connection_id}`, this.activityLogId as number, - nango, + nangoProps, syncData, this.nangoConnection.environment_id, this.writeToDb, @@ -319,7 +319,7 @@ export default class SyncRun { if (!success || (error && userDefinedResults === null)) { const message = `The integration was run but there was a problem in retrieving the results from the script "${this.syncName}"${ syncData?.version ? ` version: ${syncData.version}` : '' - }.`; + }`; await this.reportFailureForResults(message); return { success: false, error, response: false }; diff --git a/packages/worker/lib/integration.service.ts b/packages/worker/lib/integration.service.ts index 4dc05ff3eed..51267ca277f 100644 --- a/packages/worker/lib/integration.service.ts +++ b/packages/worker/lib/integration.service.ts @@ -5,6 +5,7 @@ import { createActivityLogMessage, getRootDir, NangoIntegrationData, + NangoProps, NangoSync, localFileService, remoteFileService, @@ -25,7 +26,7 @@ class IntegrationService implements IntegrationServiceInterface { syncName: string, syncId: string, activityLogId: number | undefined, - nango: NangoSync, + nangoProps: NangoProps, integrationData: NangoIntegrationData, environmentId: number, writeToDb: boolean, @@ -35,6 +36,7 @@ class IntegrationService implements IntegrationServiceInterface { temporalContext?: Context ): Promise> { try { + const nango = new NangoSync(nangoProps); const script: string | null = isCloud() && !optionalLoadLocation ? await remoteFileService.getFile(integrationData.fileLocation as string, environmentId) diff --git a/tsconfig.build.json b/tsconfig.build.json index 0697ceacf47..bc6fb6c92e3 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -6,6 +6,8 @@ { "path": "packages/worker" }, { "path": "packages/cli" }, { "path": "packages/server" }, - { "path": "packages/frontend" } + { "path": "packages/frontend" }, + { "path": "packages/runner" }, + { "path": "packages/jobs" } ] -} +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index b982c8707d1..ac9b094d39e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -7,4 +7,4 @@ "composite": true, "checkJs": false } -} +} \ No newline at end of file From 702944864f537abf90e0531df8162a6a9d89e429 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 07:53:47 -0500 Subject: [PATCH 06/85] Gh #1431 cleanup (#1432) * [gh-#1431] add gitignore for runner * [gh-#1431] use tsx to run apps --- package-lock.json | 1153 ++++++++++++++++++++++++++++++++-- package.json | 3 +- packages/jobs/nodemon.json | 2 +- packages/jobs/package.json | 1 - packages/runner/.gitignore | 3 + packages/runner/nodemon.json | 2 +- packages/server/nodemon.json | 2 +- packages/server/package.json | 1 - packages/shared/package.json | 1 - packages/worker/nodemon.json | 2 +- packages/worker/package.json | 1 - 11 files changed, 1126 insertions(+), 45 deletions(-) create mode 100644 packages/runner/.gitignore diff --git a/package-lock.json b/package-lock.json index 287d2df090b..e5fe68ccaaf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "onchange": "^7.1.0", "prettier": "^2.7.1", "testcontainers": "^9.12.0", + "tsx": "^4.6.2", "typescript": "^4.7.4", "vitest": "^0.33.0" } @@ -2912,6 +2913,54 @@ "node": ">=10.0.0" } }, + "node_modules/@esbuild/android-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz", + "integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz", + "integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz", + "integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@esbuild/darwin-arm64": { "version": "0.17.19", "cpu": [ @@ -2926,6 +2975,294 @@ "node": ">=12" } }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz", + "integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz", + "integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz", + "integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz", + "integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz", + "integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz", + "integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz", + "integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz", + "integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz", + "integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz", + "integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz", + "integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz", + "integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz", + "integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz", + "integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz", + "integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz", + "integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz", + "integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz", + "integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "dev": true, @@ -5120,8 +5457,10 @@ }, "node_modules/@tsconfig/node18-strictest-esm": { "version": "1.0.1", - "dev": true, - "license": "MIT" + "resolved": "https://registry.npmjs.org/@tsconfig/node18-strictest-esm/-/node18-strictest-esm-1.0.1.tgz", + "integrity": "sha512-cHzmAqw7CMbyqROWeBgVhard3F2V6zxOSJnQ4E6SJWruXD5ypuP9/QKekwBdfXQ4oUTaizIICKIwb+v3v33t0w==", + "deprecated": "TypeScript 5.0 supports combining TSConfigs using array syntax in extends", + "dev": true }, "node_modules/@types/amqplib": { "version": "0.8.2", @@ -7941,19 +8280,334 @@ "@esbuild/win32-x64": "0.17.19" } }, - "node_modules/escalade": { - "version": "3.1.1", - "license": "MIT", + "node_modules/esbuild/node_modules/@esbuild/android-arm": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.17.19.tgz", + "integrity": "sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "android" + ], "engines": { - "node": ">=6" + "node": ">=12" } }, - "node_modules/escape-html": { - "version": "1.0.3", - "license": "MIT" - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", + "node_modules/esbuild/node_modules/@esbuild/android-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz", + "integrity": "sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/android-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.17.19.tgz", + "integrity": "sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/darwin-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz", + "integrity": "sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/freebsd-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.19.tgz", + "integrity": "sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/freebsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.17.19.tgz", + "integrity": "sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-arm": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.17.19.tgz", + "integrity": "sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz", + "integrity": "sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-ia32": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.17.19.tgz", + "integrity": "sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-loong64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.17.19.tgz", + "integrity": "sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==", + "cpu": [ + "loong64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-mips64el": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.17.19.tgz", + "integrity": "sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==", + "cpu": [ + "mips64el" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-ppc64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.17.19.tgz", + "integrity": "sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==", + "cpu": [ + "ppc64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-riscv64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.17.19.tgz", + "integrity": "sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==", + "cpu": [ + "riscv64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-s390x": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.17.19.tgz", + "integrity": "sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/linux-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz", + "integrity": "sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/netbsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.17.19.tgz", + "integrity": "sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/openbsd-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.17.19.tgz", + "integrity": "sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/sunos-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.17.19.tgz", + "integrity": "sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/win32-arm64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz", + "integrity": "sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/win32-ia32": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.17.19.tgz", + "integrity": "sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/esbuild/node_modules/@esbuild/win32-x64": { + "version": "0.17.19", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz", + "integrity": "sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/escalade": { + "version": "3.1.1", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", "dev": true, "license": "MIT", "engines": { @@ -8861,8 +9515,10 @@ "license": "ISC" }, "node_modules/fsevents": { - "version": "2.3.2", - "license": "MIT", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, "optional": true, "os": [ "darwin" @@ -8951,6 +9607,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/get-tsconfig": { + "version": "4.7.2", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.2.tgz", + "integrity": "sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==", + "dev": true, + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, "node_modules/getopts": { "version": "2.3.0", "license": "MIT" @@ -11892,6 +12560,15 @@ "node": ">=4" } }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, "node_modules/restore-cursor": { "version": "4.0.0", "license": "MIT", @@ -13290,30 +13967,102 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, - "node_modules/tweetnacl": { - "version": "0.14.5", - "dev": true, - "license": "Unlicense" - }, - "node_modules/type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - }, - "node_modules/type-check": { - "version": "0.4.0", + "node_modules/tsx": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.6.2.tgz", + "integrity": "sha512-QPpBdJo+ZDtqZgAnq86iY/PD2KYCUPSUGIunHdGwyII99GKH+f3z3FZ8XNFLSGQIA4I365ui8wnQpl8OKLqcsg==", "dev": true, - "license": "MIT", "dependencies": { - "prelude-ls": "^1.2.1" + "esbuild": "~0.18.20", + "get-tsconfig": "^4.7.2" + }, + "bin": { + "tsx": "dist/cli.mjs" }, "engines": { - "node": ">= 0.8.0" + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" } }, - "node_modules/type-detect": { - "version": "4.0.8", - "dev": true, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz", + "integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.18.20", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz", + "integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==", + "dev": true, + "hasInstallScript": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/android-arm": "0.18.20", + "@esbuild/android-arm64": "0.18.20", + "@esbuild/android-x64": "0.18.20", + "@esbuild/darwin-arm64": "0.18.20", + "@esbuild/darwin-x64": "0.18.20", + "@esbuild/freebsd-arm64": "0.18.20", + "@esbuild/freebsd-x64": "0.18.20", + "@esbuild/linux-arm": "0.18.20", + "@esbuild/linux-arm64": "0.18.20", + "@esbuild/linux-ia32": "0.18.20", + "@esbuild/linux-loong64": "0.18.20", + "@esbuild/linux-mips64el": "0.18.20", + "@esbuild/linux-ppc64": "0.18.20", + "@esbuild/linux-riscv64": "0.18.20", + "@esbuild/linux-s390x": "0.18.20", + "@esbuild/linux-x64": "0.18.20", + "@esbuild/netbsd-x64": "0.18.20", + "@esbuild/openbsd-x64": "0.18.20", + "@esbuild/sunos-x64": "0.18.20", + "@esbuild/win32-arm64": "0.18.20", + "@esbuild/win32-ia32": "0.18.20", + "@esbuild/win32-x64": "0.18.20" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "dev": true, + "license": "Unlicense" + }, + "node_modules/type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, + "node_modules/type-check": { + "version": "0.4.0", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "dev": true, "license": "MIT", "engines": { "node": ">=4" @@ -13638,6 +14387,54 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/vite/node_modules/@esbuild/android-arm": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.12.tgz", + "integrity": "sha512-LIxaNIQfkFZbTLb4+cX7dozHlAbAshhFE5PKdro0l+FnCpx1GDJaQ2WMcqm+ToXKMt8p8Uojk/MFRuGyz3V5Sw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/android-arm64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.12.tgz", + "integrity": "sha512-BMAlczRqC/LUt2P97E4apTBbkvS9JTJnp2DKFbCwpZ8vBvXVbNdqmvzW/OsdtI/+mGr+apkkpqGM8WecLkPgrA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/android-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.12.tgz", + "integrity": "sha512-zU5MyluNsykf5cOJ0LZZZjgAHbhPJ1cWfdH1ZXVMXxVMhEV0VZiZXQdwBBVvmvbF28EizeK7obG9fs+fpmS0eQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/vite/node_modules/@esbuild/darwin-arm64": { "version": "0.18.12", "cpu": [ @@ -13653,6 +14450,294 @@ "node": ">=12" } }, + "node_modules/vite/node_modules/@esbuild/darwin-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.12.tgz", + "integrity": "sha512-ohqLPc7i67yunArPj1+/FeeJ7AgwAjHqKZ512ADk3WsE3FHU9l+m5aa7NdxXr0HmN1bjDlUslBjWNbFlD9y12Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.12.tgz", + "integrity": "sha512-GIIHtQXqgeOOqdG16a/A9N28GpkvjJnjYMhOnXVbn3EDJcoItdR58v/pGN31CHjyXDc8uCcRnFWmqaJt24AYJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/freebsd-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.12.tgz", + "integrity": "sha512-zK0b9a1/0wZY+6FdOS3BpZcPc1kcx2G5yxxfEJtEUzVxI6n/FrC2Phsxj/YblPuBchhBZ/1wwn7AyEBUyNSa6g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.12.tgz", + "integrity": "sha512-y75OijvrBE/1XRrXq1jtrJfG26eHeMoqLJ2dwQNwviwTuTtHGCojsDO6BJNF8gU+3jTn1KzJEMETytwsFSvc+Q==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-arm64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.12.tgz", + "integrity": "sha512-JKgG8Q/LL/9sw/iHHxQyVMoQYu3rU3+a5Z87DxC+wAu3engz+EmctIrV+FGOgI6gWG1z1+5nDDbXiRMGQZXqiw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ia32": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.12.tgz", + "integrity": "sha512-yoRIAqc0B4lDIAAEFEIu9ttTRFV84iuAl0KNCN6MhKLxNPfzwCBvEMgwco2f71GxmpBcTtn7KdErueZaM2rEvw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-loong64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.12.tgz", + "integrity": "sha512-qYgt3dHPVvf/MgbIBpJ4Sup/yb9DAopZ3a2JgMpNKIHUpOdnJ2eHBo/aQdnd8dJ21X/+sS58wxHtA9lEazYtXQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-mips64el": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.12.tgz", + "integrity": "sha512-wHphlMLK4ufNOONqukELfVIbnGQJrHJ/mxZMMrP2jYrPgCRZhOtf0kC4yAXBwnfmULimV1qt5UJJOw4Kh13Yfg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-ppc64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.12.tgz", + "integrity": "sha512-TeN//1Ft20ZZW41+zDSdOI/Os1bEq5dbvBvYkberB7PHABbRcsteeoNVZFlI0YLpGdlBqohEpjrn06kv8heCJg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-riscv64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.12.tgz", + "integrity": "sha512-AgUebVS4DoAblBgiB2ACQ/8l4eGE5aWBb8ZXtkXHiET9mbj7GuWt3OnsIW/zX+XHJt2RYJZctbQ2S/mDjbp0UA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-s390x": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.12.tgz", + "integrity": "sha512-dJ3Rb3Ei2u/ysSXd6pzleGtfDdc2MuzKt8qc6ls8vreP1G3B7HInX3i7gXS4BGeVd24pp0yqyS7bJ5NHaI9ing==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/linux-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.12.tgz", + "integrity": "sha512-OrNJMGQbPaVyHHcDF8ybNSwu7TDOfX8NGpXCbetwOSP6txOJiWlgQnRymfC9ocR1S0Y5PW0Wb1mV6pUddqmvmQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/netbsd-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.12.tgz", + "integrity": "sha512-55FzVCAiwE9FK8wWeCRuvjazNRJ1QqLCYGZVB6E8RuQuTeStSwotpSW4xoRGwp3a1wUsaVCdYcj5LGCASVJmMg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/openbsd-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.12.tgz", + "integrity": "sha512-qnluf8rfb6Y5Lw2tirfK2quZOBbVqmwxut7GPCIJsM8lc4AEUj9L8y0YPdLaPK0TECt4IdyBdBD/KRFKorlK3g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/sunos-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.12.tgz", + "integrity": "sha512-+RkKpVQR7bICjTOPUpkTBTaJ4TFqQBX5Ywyd/HSdDkQGn65VPkTsR/pL4AMvuMWy+wnXgIl4EY6q4mVpJal8Kg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-arm64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.12.tgz", + "integrity": "sha512-GNHuciv0mFM7ouzsU0+AwY+7eV4Mgo5WnbhfDCQGtpvOtD1vbOiRjPYG6dhmMoFyBjj+pNqQu2X+7DKn0KQ/Gw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-ia32": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.12.tgz", + "integrity": "sha512-kR8cezhYipbbypGkaqCTWIeu4zID17gamC8YTPXYtcN3E5BhhtTnwKBn9I0PJur/T6UVwIEGYzkffNL0lFvxEw==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/vite/node_modules/@esbuild/win32-x64": { + "version": "0.18.12", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.12.tgz", + "integrity": "sha512-O0UYQVkvfM/jO8a4OwoV0mAKSJw+mjWTAd1MJd/1FCX6uiMdLmMRPK/w6e9OQ0ob2WGxzIm9va/KG0Ja4zIOgg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, "node_modules/vite/node_modules/esbuild": { "version": "0.18.12", "dev": true, @@ -14360,7 +15445,6 @@ "eslint-config-prettier": "^8.3.0", "eslint-plugin-deprecation": "^1.2.1", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^5.3.2" } }, @@ -14690,7 +15774,6 @@ "@types/uuid": "^8.3.4", "@types/ws": "^8.5.4", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^4.7.4" }, "engines": { @@ -14735,7 +15818,6 @@ "rimraf": "^5.0.1", "semver": "^7.5.4", "simple-oauth2": "^5.0.0", - "ts-node": "^10.9.1", "uuid": "^9.0.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1" @@ -14958,7 +16040,6 @@ "eslint-config-prettier": "^8.3.0", "eslint-plugin-deprecation": "^1.2.1", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^4.4.2" } }, diff --git a/package.json b/package.json index 0a158184eb8..96f4e469177 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "onchange": "^7.1.0", "prettier": "^2.7.1", "testcontainers": "^9.12.0", + "tsx": "^4.6.2", "typescript": "^4.7.4", "vitest": "^0.33.0" }, @@ -70,4 +71,4 @@ "@babel/parser": "^7.22.5", "@babel/traverse": "^7.22.5" } -} \ No newline at end of file +} diff --git a/packages/jobs/nodemon.json b/packages/jobs/nodemon.json index cc4898c4d74..586aa9f3689 100644 --- a/packages/jobs/nodemon.json +++ b/packages/jobs/nodemon.json @@ -2,6 +2,6 @@ "watch": ["lib", "../shared/lib", "../../.env"], "ext": "ts,json", "ignore": ["lib/**/*.spec.ts"], - "exec": "ts-node-esm -r dotenv/config lib/app.ts dotenv_config_path=./../../.env", + "exec": "tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env", "signal": "SIGTERM" } \ No newline at end of file diff --git a/packages/jobs/package.json b/packages/jobs/package.json index ff9e92cd5c3..4572342cff0 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -46,7 +46,6 @@ "eslint-config-prettier": "^8.3.0", "eslint-plugin-deprecation": "^1.2.1", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^5.3.2" } } diff --git a/packages/runner/.gitignore b/packages/runner/.gitignore new file mode 100644 index 00000000000..8c3aba0aad2 --- /dev/null +++ b/packages/runner/.gitignore @@ -0,0 +1,3 @@ +tsconfig.tsbuildinfo +dist/* +node_modules diff --git a/packages/runner/nodemon.json b/packages/runner/nodemon.json index 2c50525bc9d..ad33404dea8 100644 --- a/packages/runner/nodemon.json +++ b/packages/runner/nodemon.json @@ -2,5 +2,5 @@ "watch": ["lib", "../shared/lib", "../../.env"], "ext": "ts,json", "ignore": ["lib/**/*.test.ts"], - "exec": "tsc && ts-node-esm -r dotenv/config lib/server.ts dotenv_config_path=./../../.env" + "exec": "tsc && tsx -r dotenv/config lib/server.ts dotenv_config_path=./../../.env" } \ No newline at end of file diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index 83bd6c1fdda..46c44855d56 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -2,5 +2,5 @@ "watch": ["lib", "../shared/lib", "../../.env", "../shared/providers.yaml"], "ext": "ts,json", "ignore": ["src/**/*.spec.ts"], - "exec": "ts-node-esm -r dotenv/config lib/server.ts Dotenv_config_path=./../../.env" + "exec": "tsx -r dotenv/config lib/server.ts Dotenv_config_path=./../../.env" } diff --git a/packages/server/package.json b/packages/server/package.json index d599123110a..87a2e5c20fa 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -72,7 +72,6 @@ "@types/uuid": "^8.3.4", "@types/ws": "^8.5.4", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^4.7.4" } } diff --git a/packages/shared/package.json b/packages/shared/package.json index e60225c8165..7fab39d4310 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -48,7 +48,6 @@ "rimraf": "^5.0.1", "semver": "^7.5.4", "simple-oauth2": "^5.0.0", - "ts-node": "^10.9.1", "uuid": "^9.0.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1" diff --git a/packages/worker/nodemon.json b/packages/worker/nodemon.json index fc12c18e11e..cc07c810f96 100644 --- a/packages/worker/nodemon.json +++ b/packages/worker/nodemon.json @@ -2,5 +2,5 @@ "watch": ["lib", "../shared/lib", "../../.env"], "ext": "ts,json", "ignore": ["lib/**/*.spec.ts"], - "exec": "ts-node-esm -r dotenv/config lib/worker.ts dotenv_config_path=./../../.env" + "exec": "tsx -r dotenv/config lib/worker.ts dotenv_config_path=./../../.env" } diff --git a/packages/worker/package.json b/packages/worker/package.json index 5103f2e7405..49ac210d2c8 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -43,7 +43,6 @@ "eslint-config-prettier": "^8.3.0", "eslint-plugin-deprecation": "^1.2.1", "nodemon": "^3.0.1", - "ts-node": "^10.9.1", "typescript": "^4.4.2" } } From b5e0ab3cc5d3c6f16e7e3ae7738de91e04cba398 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Mon, 18 Dec 2023 14:15:36 +0100 Subject: [PATCH 07/85] temporal worker: set maxConcurrentWorkflowTaskExecutions to 50 (#1433) --- packages/jobs/lib/temporal.ts | 3 ++- packages/worker/lib/worker.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts index 8bf51ad0535..eb53ce0ed80 100644 --- a/packages/jobs/lib/temporal.ts +++ b/packages/jobs/lib/temporal.ts @@ -46,7 +46,8 @@ export class Temporal { namespace: this.namespace, workflowsPath: createRequire(import.meta.url).resolve('./workflows'), activities, - taskQueue: TASK_QUEUE + taskQueue: TASK_QUEUE, + maxConcurrentWorkflowTaskExecutions: 50 }); // Worker connects to localhost by default and uses console.error for logging. // Customize the Worker by passing more options to create(): diff --git a/packages/worker/lib/worker.ts b/packages/worker/lib/worker.ts index d965f84103f..fd5d8fe9804 100644 --- a/packages/worker/lib/worker.ts +++ b/packages/worker/lib/worker.ts @@ -37,7 +37,8 @@ async function run() { namespace, workflowsPath: createRequire(import.meta.url).resolve('./workflows'), activities, - taskQueue: TASK_QUEUE + taskQueue: TASK_QUEUE, + maxConcurrentWorkflowTaskExecutions: 50 }); // Worker connects to localhost by default and uses console.error for logging. // Customize the Worker by passing more options to create(): From 74f8aab6b7b0b1cec17a4bc23e2aa29954573a32 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 14:59:51 -0500 Subject: [PATCH 08/85] [NAN-20] implement hubspot webhooks (#1407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [nan-20] parse new properties and set validation logic • add tests to support new properties * [nan-20] replaceMetada (set) and updateMetada • update docs • adds tests * use connectionCreatedHook to make it easier to manage connection lifecycle * [nan-20] progress on webhook and post connection * [nan-20] webhook logic written in the sync script * [nan-20] allow a webhook update to send a nango webhook once finished * [nan-20] feedback updates * [nan-20] fix spacing * [nan-20] update test failure * [nan-20] remove todo * [nan-20] don't allow invalid webhooks through * [nan-20] query for the connection using the identifier instead of looping over all connections * [nan-20] observability * [nan-20] add jira webhook handling * [nan-20] pr feedback * [nan-20] remove deprecated * [nan-20] remove runner node_modules and dist * [nan-20] update webhook logic * [nan-20] add possibility for customer defined feature flag * [nan-20] remove tsbuild file * [nan-20] strip extra connection information --- .../api-reference/connection/set-metadata.mdx | 8 +- .../connection/update-metadata.mdx | 16 + docs-v2/mint.json | 1 + docs-v2/spec.yaml | 38 +++ .../hubspot/hubspot-contacts.ts | 25 ++ integration-templates/hubspot/nango.yaml | 11 + packages/cli/lib/cli.ts | 3 +- .../nango-yaml/v2/invalid.1/nango.yaml | 76 +++++ .../nango-yaml/v2/invalid.2/nango.yaml | 79 +++++ .../cli/lib/fixtures/nango-yaml/v2/nango.yaml | 3 +- .../fixtures/nango-yaml/v2/valid/nango.yaml | 77 +++++ .../nango-yaml/v2/{ => valid}/object.json | 9 +- packages/cli/lib/nango.yaml.schema.v2.json | 18 +- packages/cli/lib/services/compile.service.ts | 6 +- packages/cli/lib/services/deploy.service.ts | 3 +- .../lib/services/local-integration.service.ts | 16 +- .../cli/lib/services/verification.service.ts | 4 +- packages/cli/lib/sync.unit.test.ts | 18 +- packages/cli/lib/templates/sync.ejs | 6 + packages/jobs/lib/activities.ts | 66 +++- packages/jobs/lib/integration.service.ts | 18 +- packages/jobs/lib/models/worker.ts | 8 + packages/jobs/lib/temporal.ts | 36 +- packages/jobs/lib/workflows.ts | 14 +- packages/node-client/lib/index.ts | 22 ++ packages/runner/lib/client.unit.test.ts | 5 +- packages/runner/lib/exec.ts | 33 +- packages/runner/lib/server.ts | 5 +- .../lib/controllers/apiAuth.controller.ts | 24 +- .../lib/controllers/appAuth.controller.ts | 13 +- .../config.controller.integration.test.ts | 1 + .../lib/controllers/config.controller.ts | 4 +- .../lib/controllers/connection.controller.ts | 78 ++++- .../lib/controllers/environment.controller.ts | 3 + .../lib/controllers/oauth.controller.ts | 13 +- .../lib/controllers/unauth.controller.ts | 13 +- .../lib/controllers/webhook.controller.ts | 32 ++ packages/server/lib/server.ts | 3 + packages/shared/lib/clients/sync.client.ts | 130 ++++++- packages/shared/lib/constants.ts | 3 +- ...6_add_webhooks_subscriptions_to_config.cjs | 13 + ...20231205151403_add_uuid_to_environment.cjs | 13 + .../20231208181152_update_jobs_type_enum.cjs | 31 ++ packages/shared/lib/hooks/hooks.ts | 10 + packages/shared/lib/index.ts | 4 + .../scripts/connection/connection.manager.ts | 131 +++++++ .../connection/hubspot-post-connection.ts | 12 + .../integrations/scripts/connection/index.ts | 2 + .../connection/jira-post-connection.ts | 26 ++ .../webhook/hubspot-webhook-routing.ts | 28 ++ .../lib/integrations/scripts/webhook/index.ts | 2 + .../scripts/webhook/jira-webhook-routing.ts | 12 + .../scripts/webhook/webhook.manager.ts | 105 ++++++ packages/shared/lib/models/Activity.ts | 6 +- packages/shared/lib/models/Connection.ts | 11 +- packages/shared/lib/models/Environment.ts | 3 + packages/shared/lib/models/NangoConfig.ts | 2 + packages/shared/lib/models/Provider.ts | 2 + packages/shared/lib/models/Sync.ts | 7 +- packages/shared/lib/sdk/sync.ts | 116 ++++++- .../shared/lib/services/config.service.ts | 18 + .../connection.service.integration.test.ts | 59 ++++ .../shared/lib/services/connection.service.ts | 85 ++++- .../lib/services/environment.service.ts | 24 ++ .../lib/services/nango-config.service.ts | 15 +- .../services/sync/config/config.service.ts | 12 +- .../services/sync/config/deploy.service.ts | 322 ++++++++++-------- .../lib/services/sync/data/data.service.ts | 88 ++++- .../lib/services/sync/data/records.service.ts | 42 +++ .../sync/notification/webhook.service.ts | 75 +++- .../shared/lib/services/sync/run.service.ts | 79 +++-- packages/shared/lib/utils/error.ts | 4 + packages/shared/lib/utils/metrics.manager.ts | 4 +- packages/shared/lib/utils/utils.ts | 5 + packages/shared/providers.yaml | 9 + packages/webapp/src/pages/Activity.tsx | 15 + .../webapp/src/pages/IntegrationCreate.tsx | 29 ++ packages/webapp/src/types.ts | 2 +- packages/worker/lib/activities.ts | 62 +++- packages/worker/lib/integration.service.ts | 60 +++- packages/worker/lib/models/Worker.ts | 8 + packages/worker/lib/worker.ts | 31 +- packages/worker/lib/workflows.ts | 14 +- 83 files changed, 2210 insertions(+), 329 deletions(-) create mode 100644 docs-v2/api-reference/connection/update-metadata.mdx create mode 100644 integration-templates/hubspot/hubspot-contacts.ts create mode 100644 packages/cli/lib/fixtures/nango-yaml/v2/invalid.1/nango.yaml create mode 100644 packages/cli/lib/fixtures/nango-yaml/v2/invalid.2/nango.yaml create mode 100644 packages/cli/lib/fixtures/nango-yaml/v2/valid/nango.yaml rename packages/cli/lib/fixtures/nango-yaml/v2/{ => valid}/object.json (97%) create mode 100644 packages/server/lib/controllers/webhook.controller.ts create mode 100644 packages/shared/lib/db/migrations/20231204191116_add_webhooks_subscriptions_to_config.cjs create mode 100644 packages/shared/lib/db/migrations/20231205151403_add_uuid_to_environment.cjs create mode 100644 packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs create mode 100644 packages/shared/lib/hooks/hooks.ts create mode 100644 packages/shared/lib/integrations/scripts/connection/connection.manager.ts create mode 100644 packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts create mode 100644 packages/shared/lib/integrations/scripts/connection/index.ts create mode 100644 packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts create mode 100644 packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts create mode 100644 packages/shared/lib/integrations/scripts/webhook/index.ts create mode 100644 packages/shared/lib/integrations/scripts/webhook/jira-webhook-routing.ts create mode 100644 packages/shared/lib/integrations/scripts/webhook/webhook.manager.ts create mode 100644 packages/shared/lib/services/connection.service.integration.test.ts diff --git a/docs-v2/api-reference/connection/set-metadata.mdx b/docs-v2/api-reference/connection/set-metadata.mdx index 8e5fcb392ed..b9a6bf533e8 100644 --- a/docs-v2/api-reference/connection/set-metadata.mdx +++ b/docs-v2/api-reference/connection/set-metadata.mdx @@ -1,14 +1,14 @@ --- -title: 'Update connection metadata' +title: 'Set connection metadata' openapi: 'POST /connection/{connectionId}/metadata' --- -## Update _connection metadata_ +## Set _connection metadata_ -Use this API endpoint to update your [custom metadata](/guides/advanced-auth#storing-custom-metadata-per-connection) for a _connection_. +Use this API endpoint to set your [custom metadata](/guides/advanced-auth#storing-custom-metadata-per-connection) for a _connection_. Nango uses the request body as the new metadata (it must be a JSON object). Note that this overrides any existing metadata. ## Fetching _connection metadata_ -To read the existing metadata of a _connection_, simply [fetch it](/api-reference/connection/get). Your custom metadata is included as part of the returned _connection_ object. \ No newline at end of file +To read the existing metadata of a _connection_, simply [fetch it](/api-reference/connection/get). Your custom metadata is included as part of the returned _connection_ object.To read the existing metadata of a _connection_, simply [fetch it](/api-reference/connection/get). Your custom metadata is included as part of the returned _connection_ object. diff --git a/docs-v2/api-reference/connection/update-metadata.mdx b/docs-v2/api-reference/connection/update-metadata.mdx new file mode 100644 index 00000000000..419b4dd9b4b --- /dev/null +++ b/docs-v2/api-reference/connection/update-metadata.mdx @@ -0,0 +1,16 @@ +--- +title: 'Update connection metadata' +openapi: 'PATCH /connection/{connectionId}/metadata' +--- + +## Update _connection metadata_ + +Use this API endpoint to update your [custom metadata](/guides/advanced-auth#storing-custom-metadata-per-connection) for a _connection_. + +Nango uses the request body as the new metadata (it must be a JSON object). This will take in whatever metadata and merge it with any +existing metadata. + +## Fetching _connection metadata_ + +To read the existing metadata of a _connection_, simply [fetch it](/api-reference/connection/get). Your custom metadata is included as part of the returned _connection_ object. + diff --git a/docs-v2/mint.json b/docs-v2/mint.json index 5c8e9d77a72..6a113343d5e 100644 --- a/docs-v2/mint.json +++ b/docs-v2/mint.json @@ -340,6 +340,7 @@ "api-reference/connection/get", "api-reference/connection/post", "api-reference/connection/set-metadata", + "api-reference/connection/update-metadata", "api-reference/connection/delete" ] }, diff --git a/docs-v2/spec.yaml b/docs-v2/spec.yaml index d03d2374386..8226b4bc4de 100644 --- a/docs-v2/spec.yaml +++ b/docs-v2/spec.yaml @@ -509,6 +509,44 @@ paths: message: type: string + patch: + description: Update custom metadata for the connection. + parameters: + - name: connectionId + in: path + required: true + schema: + type: string + description: The connection ID used to create the connection. + - name: Provider-Config-Key + in: header + required: true + description: The integration ID used to create the connection (aka Unique Key). + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + responses: + '200': + description: Successfully updated the metadata + content: + application/json: + schema: + type: object + '400': + description: Invalid request + content: + application/json: + schema: + type: object + properties: + message: + type: string + /records: get: description: Returns data synced with Nango Sync diff --git a/integration-templates/hubspot/hubspot-contacts.ts b/integration-templates/hubspot/hubspot-contacts.ts new file mode 100644 index 00000000000..3b1dc94c1ba --- /dev/null +++ b/integration-templates/hubspot/hubspot-contacts.ts @@ -0,0 +1,25 @@ +import type { NangoSync, HubspotContact } from './models'; + +export default async function fetchData(nango: NangoSync) { + const query = `properties=firstname,lastname,email`; + + for await (const records of nango.paginate({ endpoint: '/crm/v3/objects/contacts', params: { query } })) { + const mappedRecords = mapHubspotContacts(records); + + await nango.batchSave(mappedRecords, 'HubspotContact'); + } +} + +function mapHubspotContacts(records: any[]): HubspotContact[] { + return records.map((record: any) => { + return { + id: record.id as string, + created_at: record.createdAt, + updated_at: record.updatedAt, + first_name: record.properties.firstname, + last_name: record.properties.lastname, + email: record.properties.email, + active: record.archived !== true + }; + }); +} diff --git a/integration-templates/hubspot/nango.yaml b/integration-templates/hubspot/nango.yaml index f3784a1ca33..b245ec7b317 100644 --- a/integration-templates/hubspot/nango.yaml +++ b/integration-templates/hubspot/nango.yaml @@ -4,6 +4,10 @@ integrations: runs: every half hour returns: - HubspotServiceTicket + hubspot-contacts: + runs: every day + returns: + - HubspotContact hubspot-owner: runs: every day returns: @@ -53,3 +57,10 @@ models: category: string content: string publishDate: number + HubspotContact: + id: string + created_at: string + updated_at: string + first_name: string + last_name: string + email: string diff --git a/packages/cli/lib/cli.ts b/packages/cli/lib/cli.ts index 0ac97f05a83..d5de95ebbe4 100644 --- a/packages/cli/lib/cli.ts +++ b/packages/cli/lib/cli.ts @@ -146,7 +146,8 @@ export const generate = async (debug = false, inParentDirectory = false) => { interfaceFileName: TYPES_FILE_NAME.replace('.ts', ''), interfaceNames, mappings, - inputs: input && Object.keys(input).length > 0 ? input : '' + inputs: input && Object.keys(input).length > 0 ? input : '', + hasWebhook: type === SyncConfigType.SYNC && flow.webhookSubscriptions && flow.webhookSubscriptions.length > 0 }); const stripped = rendered.replace(/^\s+/, ''); diff --git a/packages/cli/lib/fixtures/nango-yaml/v2/invalid.1/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v2/invalid.1/nango.yaml new file mode 100644 index 00000000000..156834fae02 --- /dev/null +++ b/packages/cli/lib/fixtures/nango-yaml/v2/invalid.1/nango.yaml @@ -0,0 +1,76 @@ +integrations: + demo-github-integration: + syncs: + github-issue-example: + description: | + Sync github issues continuously from public repos + sync_type: incremental + auto_start: true + runs: every half hour + scopes: public_repo + output: GithubIssue + webhook-subscriptions: + - issue.creation + github-issue-example-two: + description: | + Sync github issues continuously from public repos example two + sync_type: incremental + auto_start: true + runs: every hour + endpoint: /ticketing/tickets-two + scopes: public_repo + output: GithubIssue2 + github-multiple-models: + description: Sync github issues to multiple models + sync_type: full + auto_start: true + runs: every 5 minutes + endpoint: + - /ticketing/ticket + - /ticketing/pr + output: + - GithubIssue3 + - GithubPR + actions: + github-get-issue: + description: Get a GitHub issue. + endpoint: GET /ticketing/tickets/{GithubCreateIssueInput:id} + output: GithubIssueAction + scopes: repo:read + github-create-issue: + description: Creates a GitHub issue. + endpoint: /ticketing/tickets + scopes: repo:write + input: GithubCreateIssueInput + output: GithubCreateOutput + github-delete-issue: + description: Deletes a GitHub issue. + endpoint: DELETE /ticketing/tickets/{GithubIssue:id} + scopes: repo:write + output: boolean + +models: + GithubIssue: + id: integer + owner: string + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string + GithubIssueAction: + __extends: GithubIssue + GithubIssue2: + __extends: GithubIssue + GithubIssue3: + __extends: GithubIssue + GithubCreateIssueInput: + __extends: GithubIssue + GithubCreateOutput: + result: GithubIssue + GithubPR: + __extends: GithubIssue diff --git a/packages/cli/lib/fixtures/nango-yaml/v2/invalid.2/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v2/invalid.2/nango.yaml new file mode 100644 index 00000000000..7d9f90b9cfa --- /dev/null +++ b/packages/cli/lib/fixtures/nango-yaml/v2/invalid.2/nango.yaml @@ -0,0 +1,79 @@ +integrations: + demo-github-integration: + syncs: + github-issue-example: + description: | + Sync github issues continuously from public repos + sync_type: incremental + auto_start: true + runs: every half hour + endpoint: DELETE /ticketing/tickets + scopes: public_repo + output: GithubIssue + webhook-subscriptions: + - issue.creation + github-issue-example-two: + description: | + Sync github issues continuously from public repos example two + sync_type: incremental + auto_start: true + runs: every hour + endpoint: /ticketing/tickets-two + scopes: public_repo + output: GithubIssue2 + github-multiple-models: + description: Sync github issues to multiple models + sync_type: full + auto_start: true + runs: every 5 minutes + endpoint: + - /ticketing/ticket + - /ticketing/pr + output: + - GithubIssue3 + - GithubPR + actions: + github-get-issue: + description: Get a GitHub issue. + endpoint: GET /ticketing/tickets/{GithubCreateIssueInput:id} + output: GithubIssueAction + scopes: repo:read + webhook-subscriptions: + - issue.creation + github-create-issue: + description: Creates a GitHub issue. + endpoint: /ticketing/tickets + scopes: repo:write + input: GithubCreateIssueInput + output: GithubCreateOutput + github-delete-issue: + description: Deletes a GitHub issue. + endpoint: DELETE /ticketing/tickets/{GithubIssue:id} + scopes: repo:write + output: boolean + +models: + GithubIssue: + id: integer + owner: string + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string + GithubIssueAction: + __extends: GithubIssue + GithubIssue2: + __extends: GithubIssue + GithubIssue3: + __extends: GithubIssue + GithubCreateIssueInput: + __extends: GithubIssue + GithubCreateOutput: + result: GithubIssue + GithubPR: + __extends: GithubIssue diff --git a/packages/cli/lib/fixtures/nango-yaml/v2/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v2/nango.yaml index 62ba925b313..156834fae02 100644 --- a/packages/cli/lib/fixtures/nango-yaml/v2/nango.yaml +++ b/packages/cli/lib/fixtures/nango-yaml/v2/nango.yaml @@ -7,9 +7,10 @@ integrations: sync_type: incremental auto_start: true runs: every half hour - endpoint: DELETE /ticketing/tickets scopes: public_repo output: GithubIssue + webhook-subscriptions: + - issue.creation github-issue-example-two: description: | Sync github issues continuously from public repos example two diff --git a/packages/cli/lib/fixtures/nango-yaml/v2/valid/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v2/valid/nango.yaml new file mode 100644 index 00000000000..b012c53b189 --- /dev/null +++ b/packages/cli/lib/fixtures/nango-yaml/v2/valid/nango.yaml @@ -0,0 +1,77 @@ +integrations: + demo-github-integration: + syncs: + github-issue-example: + description: | + Sync github issues continuously from public repos + sync_type: incremental + auto_start: true + runs: every half hour + endpoint: DELETE /ticketing/tickets + scopes: public_repo + output: GithubIssue + webhook-subscriptions: + - issue.creation + github-issue-example-two: + description: | + Sync github issues continuously from public repos example two + sync_type: incremental + auto_start: true + runs: every hour + endpoint: /ticketing/tickets-two + scopes: public_repo + output: GithubIssue2 + github-multiple-models: + description: Sync github issues to multiple models + sync_type: full + auto_start: true + runs: every 5 minutes + endpoint: + - /ticketing/ticket + - /ticketing/pr + output: + - GithubIssue3 + - GithubPR + actions: + github-get-issue: + description: Get a GitHub issue. + endpoint: GET /ticketing/tickets/{GithubCreateIssueInput:id} + output: GithubIssueAction + scopes: repo:read + github-create-issue: + description: Creates a GitHub issue. + endpoint: /ticketing/tickets + scopes: repo:write + input: GithubCreateIssueInput + output: GithubCreateOutput + github-delete-issue: + description: Deletes a GitHub issue. + endpoint: DELETE /ticketing/tickets/{GithubIssue:id} + scopes: repo:write + output: boolean + +models: + GithubIssue: + id: integer + owner: string + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string + GithubIssueAction: + __extends: GithubIssue + GithubIssue2: + __extends: GithubIssue + GithubIssue3: + __extends: GithubIssue + GithubCreateIssueInput: + __extends: GithubIssue + GithubCreateOutput: + result: GithubIssue + GithubPR: + __extends: GithubIssue diff --git a/packages/cli/lib/fixtures/nango-yaml/v2/object.json b/packages/cli/lib/fixtures/nango-yaml/v2/valid/object.json similarity index 97% rename from packages/cli/lib/fixtures/nango-yaml/v2/object.json rename to packages/cli/lib/fixtures/nango-yaml/v2/valid/object.json index a383001ab68..89787f10e2b 100644 --- a/packages/cli/lib/fixtures/nango-yaml/v2/object.json +++ b/packages/cli/lib/fixtures/nango-yaml/v2/valid/object.json @@ -32,7 +32,8 @@ "returns": ["GithubIssue"], "description": "Sync github issues continuously from public repos\n", "scopes": ["public_repo"], - "endpoints": [{ "GET": "/ticketing/tickets" }] + "endpoints": [{ "GET": "/ticketing/tickets" }], + "webhookSubscriptions": ["issue.creation"] }, { "name": "github-issue-example-two", @@ -64,7 +65,8 @@ "returns": ["GithubIssue2"], "description": "Sync github issues continuously from public repos example two\n", "scopes": ["public_repo"], - "endpoints": [{ "GET": "/ticketing/tickets-two" }] + "endpoints": [{ "GET": "/ticketing/tickets-two" }], + "webhookSubscriptions": [] }, { "name": "github-multiple-models", @@ -112,7 +114,8 @@ "returns": ["GithubIssue3", "GithubPR"], "description": "Sync github issues to multiple models", "scopes": [], - "endpoints": [{ "GET": "/ticketing/ticket" }, { "GET": "/ticketing/pr" }] + "endpoints": [{ "GET": "/ticketing/ticket" }, { "GET": "/ticketing/pr" }], + "webhookSubscriptions": [] } ], "actions": [ diff --git a/packages/cli/lib/nango.yaml.schema.v2.json b/packages/cli/lib/nango.yaml.schema.v2.json index c953d2e3e60..1fdd70acd22 100644 --- a/packages/cli/lib/nango.yaml.schema.v2.json +++ b/packages/cli/lib/nango.yaml.schema.v2.json @@ -97,6 +97,18 @@ "errorMessage": { "_": "nango yaml schema validation error: output must be a string or an array of strings." } + }, + "webhook-subscriptions": { + "oneOf": [ + { "type": "string" }, + { + "type": "array", + "items": { "type": "string" } + } + ], + "errorMessage": { + "_": "nango yaml schema validation error: webhook-subscriptions must be a string or an array of strings." + } } }, "required": ["endpoint", "output"], @@ -181,7 +193,11 @@ "errorMessage": { "required": { "endpoint": "nango yaml schema validation error: An endpoint property is required to specify how to trigger the action." - } + }, + "_": "nango yaml schema validation error: webhook-subscriptions is not a valid property for an action." + }, + "not": { + "required": ["webhook-subscriptions"] } } } diff --git a/packages/cli/lib/services/compile.service.ts b/packages/cli/lib/services/compile.service.ts index 16958a42a83..cd9823ba8c7 100644 --- a/packages/cli/lib/services/compile.service.ts +++ b/packages/cli/lib/services/compile.service.ts @@ -3,7 +3,7 @@ import * as tsNode from 'ts-node'; import glob from 'glob'; import chalk from 'chalk'; import path from 'path'; -import { SyncConfigType } from '@nangohq/shared'; +import { SyncConfigType, NangoSyncConfig } from '@nangohq/shared'; import configService from './config.service.js'; import { getNangoRootPath, printDebug } from '../utils.js'; @@ -55,10 +55,12 @@ class CompileService { const providerConfiguration = config.find((config) => [...config.syncs, ...config.actions].find((sync) => sync.name === path.basename(filePath, '.ts')) ); + if (!providerConfiguration) { continue; } - const syncConfig = [...providerConfiguration.syncs, ...providerConfiguration.actions].find( + + const syncConfig = [...(providerConfiguration?.syncs as NangoSyncConfig[]), ...(providerConfiguration?.actions as NangoSyncConfig[])].find( (sync) => sync.name === path.basename(filePath, '.ts') ); const type = syncConfig?.type || SyncConfigType.SYNC; diff --git a/packages/cli/lib/services/deploy.service.ts b/packages/cli/lib/services/deploy.service.ts index c7370f38558..2a1c16a560b 100644 --- a/packages/cli/lib/services/deploy.service.ts +++ b/packages/cli/lib/services/deploy.service.ts @@ -295,7 +295,8 @@ class DeployService { ts: localFileService.getIntegrationTsFile(syncName, './') as string }, model_schema: JSON.stringify(model_schema), - endpoints: flow.endpoints + endpoints: flow.endpoints, + webhookSubscriptions: flow.webhookSubscriptions || [] }; postData.push(body); diff --git a/packages/cli/lib/services/local-integration.service.ts b/packages/cli/lib/services/local-integration.service.ts index 111ccf6df54..23251df0168 100644 --- a/packages/cli/lib/services/local-integration.service.ts +++ b/packages/cli/lib/services/local-integration.service.ts @@ -13,7 +13,8 @@ class IntegrationService implements IntegrationServiceInterface { _integrationData: NangoIntegrationData, _environmentId: number, _writeToDb: boolean, - isAction: boolean, + isInvokedImmediately: boolean, + isWebhook: boolean, optionalLoadLocation?: string, input?: object ): Promise { @@ -33,7 +34,7 @@ class IntegrationService implements IntegrationServiceInterface { var module = { exports: {} }; var exports = module.exports; ${script} - return module.exports.default || module.exports; + return module.exports; })(); `; @@ -56,8 +57,8 @@ class IntegrationService implements IntegrationServiceInterface { const context = vm.createContext(sandbox); const scriptExports: any = scriptObj.runInContext(context); - if (scriptExports && typeof scriptExports === 'function') { - const results = isAction ? await scriptExports(nango, input) : await scriptExports(nango); + if (scriptExports.default && typeof scriptExports.default === 'function') { + const results = isInvokedImmediately ? await scriptExports.default(nango, input) : await scriptExports.default(nango); return { success: true, error: null, response: results }; } else { const content = `There is no default export that is a function for ${syncName}`; @@ -65,7 +66,12 @@ class IntegrationService implements IntegrationServiceInterface { return { success: false, error: new NangoError(content, 500), response: null }; } } catch (err: any) { - const errorType = isAction ? 'action_script_failure' : 'sync_script_failure'; + let errorType = 'sync_script_failure'; + if (isWebhook) { + errorType = 'webhook_script_failure'; + } else if (isInvokedImmediately) { + errorType = 'action_script_failure'; + } return formatScriptError(err, errorType, syncName); } diff --git a/packages/cli/lib/services/verification.service.ts b/packages/cli/lib/services/verification.service.ts index ecd57392100..3f98378ebd6 100644 --- a/packages/cli/lib/services/verification.service.ts +++ b/packages/cli/lib/services/verification.service.ts @@ -103,13 +103,13 @@ class VerificationService { const syncNames = config.map((provider) => provider.syncs.map((sync) => sync.name)).flat(); const actionNames = config.map((provider) => provider.actions.map((action) => action.name)).flat(); - const flows = [...syncNames, ...actionNames]; + const flows = [...syncNames, ...actionNames].filter((name) => name); const tsFiles = glob.sync(`./*.ts`); const tsFileNames = tsFiles.filter((file) => !file.includes('models.ts')).map((file) => path.basename(file, '.ts')); - const missingSyncsAndActions = flows.filter((syncOrActionName) => !tsFileNames.includes(syncOrActionName)); + const missingSyncsAndActions = flows.filter((syncOrActionName) => !tsFileNames.includes(syncOrActionName as string)); if (missingSyncsAndActions.length > 0) { console.log(chalk.red(`The following syncs are missing a corresponding .ts file: ${missingSyncsAndActions.join(', ')}`)); diff --git a/packages/cli/lib/sync.unit.test.ts b/packages/cli/lib/sync.unit.test.ts index 3e6a3eba7d7..cd13b155171 100644 --- a/packages/cli/lib/sync.unit.test.ts +++ b/packages/cli/lib/sync.unit.test.ts @@ -320,9 +320,23 @@ describe('generate function tests', () => { }); it('should parse a nango.yaml file that is version 2 as expected', async () => { - const { response: config } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v2`)); + const { response: config } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v2/valid`)); expect(config).toBeDefined(); - const json = fs.readFileSync(path.resolve(__dirname, `./fixtures/nango-yaml/v2/object.json`), 'utf8'); + const json = fs.readFileSync(path.resolve(__dirname, `./fixtures/nango-yaml/v2/valid/object.json`), 'utf8'); expect(config).toEqual(JSON.parse(json)); }); + + it('should throw a validation error on a nango.yaml file that is not formatted correctly -- missing endpoint', async () => { + const { response: config, error } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v2/invalid.1`)); + expect(config).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toEqual('Problem validating the nango.yaml file.'); + }); + + it('should throw a validation error on a nango.yaml file that is not formatted correctly -- webhook subscriptions are not allowed in an action', async () => { + const { response: config, error } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v2/invalid.2`)); + expect(config).toBeNull(); + expect(error).toBeDefined(); + expect(error?.message).toEqual('Problem validating the nango.yaml file.'); + }); }); diff --git a/packages/cli/lib/templates/sync.ejs b/packages/cli/lib/templates/sync.ejs index e3467539709..0b5fe15397a 100644 --- a/packages/cli/lib/templates/sync.ejs +++ b/packages/cli/lib/templates/sync.ejs @@ -8,3 +8,9 @@ export default async function fetchData(nango: NangoSync): Promise { // to save data use await nango.batchSave // to delete data use await nango.batchDelete } +<% if (hasWebhook) { %> +export async function onWebhookPayloadReceived(nango: NangoSync, payload: any): Promise { + // handle the subscription types you need + // use nango.batchUpdate for object change events +} +<% } %> diff --git a/packages/jobs/lib/activities.ts b/packages/jobs/lib/activities.ts index dda22830920..9f0e3281cc4 100644 --- a/packages/jobs/lib/activities.ts +++ b/packages/jobs/lib/activities.ts @@ -23,10 +23,11 @@ import { MetricTypes, isInitialSyncStillRunning, initialSyncExists, + getSyncByIdAndName, logger } from '@nangohq/shared'; import integrationService from './integration.service.js'; -import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/worker'; +import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs, WebhookArgs } from './models/worker'; export async function routeSync(args: InitialSyncArgs): Promise { const { syncId, syncJobId, syncName, activityLogId, nangoConnection, debug } = args; @@ -307,23 +308,76 @@ export async function syncProvider( } } +export async function runWebhook(args: WebhookArgs): Promise { + const { input, nangoConnection, activityLogId, parentSyncName } = args; + + const syncConfig: ProviderConfig = (await configService.getProviderConfig( + nangoConnection?.provider_config_key as string, + nangoConnection?.environment_id as number + )) as ProviderConfig; + + const sync = await getSyncByIdAndName(nangoConnection.id as number, parentSyncName); + + const context: Context = Context.current(); + + const syncJobId = await createSyncJob( + sync?.id as string, + SyncType.WEBHOOK, + SyncStatus.RUNNING, + context.info.workflowExecution.workflowId, + nangoConnection, + context.info.workflowExecution.runId + ); + + const syncRun = new syncRunService({ + integrationService, + writeToDb: true, + nangoConnection, + syncJobId: syncJobId?.id as number, + syncName: parentSyncName, + isAction: false, + syncType: SyncType.WEBHOOK, + isWebhook: true, + activityLogId, + input, + provider: syncConfig.provider, + debug: false, + temporalContext: context + }); + + const result = await syncRun.run(); + + return result.success; +} + export async function reportFailure( error: any, - workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs, + workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs | WebhookArgs, DEFAULT_TIMEOUT: string, MAXIMUM_ATTEMPTS: number ): Promise { const { nangoConnection } = workflowArguments; - const type = 'syncName' in workflowArguments ? 'sync' : 'action'; - const name = 'syncName' in workflowArguments ? workflowArguments.syncName : workflowArguments.actionName; + let type = 'webhook'; + + let name = ''; + if ('syncName' in workflowArguments) { + name = workflowArguments.syncName; + type = 'sync'; + } else if ('actionName' in workflowArguments) { + name = workflowArguments.actionName; + type = 'action'; + } else { + name = workflowArguments.name; + } + let content = `The ${type} "${name}" failed `; const context: Context = Context.current(); if (error instanceof CancelledFailure) { content += `due to a cancellation.`; - } else if (error.cause instanceof TerminatedFailure || error.cause.name === 'TerminatedFailure') { + } else if (error.cause instanceof TerminatedFailure || error.cause?.name === 'TerminatedFailure') { content += `due to a termination.`; - } else if (error.cause instanceof TimeoutFailure || error.cause.name === 'TimeoutFailure') { + } else if (error.cause instanceof TimeoutFailure || error.cause?.name === 'TimeoutFailure') { if (error.cause.timeoutType === 3) { content += `due to a timeout with respect to the max schedule length timeout of ${DEFAULT_TIMEOUT}.`; } else { diff --git a/packages/jobs/lib/integration.service.ts b/packages/jobs/lib/integration.service.ts index 4c40b284679..6a284785e22 100644 --- a/packages/jobs/lib/integration.service.ts +++ b/packages/jobs/lib/integration.service.ts @@ -31,7 +31,8 @@ class IntegrationService implements IntegrationServiceInterface { integrationData: NangoIntegrationData, environmentId: number, writeToDb: boolean, - isAction: boolean, + isInvokedImmediately: boolean, + isWebhook: boolean, optionalLoadLocation?: string, input?: object, temporalContext?: Context @@ -95,10 +96,21 @@ class IntegrationService implements IntegrationServiceInterface { try { // TODO: request sent to the runner for it to run the script is synchronous. // TODO: Make the request return immediately and have the runner ping the job service when it's done. - const res = await runner.client.run.mutate({ nangoProps, code: script as string, codeParams: input as object, isAction }); + const res = await runner.client.run.mutate({ + nangoProps, + code: script as string, + codeParams: input as object, + isInvokedImmediately, + isWebhook + }); return { success: true, error: null, response: res }; } catch (err: any) { - const errorType = isAction ? 'action_script_failure' : 'sync_script_failure'; + let errorType = 'sync_script_failure'; + if (isWebhook) { + errorType = 'webhook_script_failure'; + } else if (isInvokedImmediately) { + errorType = 'action_script_failure'; + } const { success, error, response } = formatScriptError(err, errorType, syncName); if (activityLogId && writeToDb) { diff --git a/packages/jobs/lib/models/worker.ts b/packages/jobs/lib/models/worker.ts index fafa9297ec9..3cae8d7a052 100644 --- a/packages/jobs/lib/models/worker.ts +++ b/packages/jobs/lib/models/worker.ts @@ -24,3 +24,11 @@ export interface ActionArgs { nangoConnection: NangoConnection; activityLogId: number; } + +export interface WebhookArgs { + name: string; + parentSyncName: string; + nangoConnection: NangoConnection; + input: object; + activityLogId: number; +} diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts index eb53ce0ed80..50ff9ec6d6c 100644 --- a/packages/jobs/lib/temporal.ts +++ b/packages/jobs/lib/temporal.ts @@ -3,15 +3,15 @@ import fs from 'fs-extra'; import * as dotenv from 'dotenv'; import { createRequire } from 'module'; import * as activities from './activities.js'; -import { TASK_QUEUE, isProd } from '@nangohq/shared'; +import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd } from '@nangohq/shared'; export class Temporal { namespace: string; - worker: Worker | null; + workers: Worker[] | null; constructor(namespace: string) { this.namespace = namespace; - this.worker = null; + this.workers = null; } async start() { @@ -41,27 +41,33 @@ export class Temporal { } }); - this.worker = await Worker.create({ + const syncWorker = { connection, namespace: this.namespace, workflowsPath: createRequire(import.meta.url).resolve('./workflows'), activities, - taskQueue: TASK_QUEUE, - maxConcurrentWorkflowTaskExecutions: 50 - }); - // Worker connects to localhost by default and uses console.error for logging. - // Customize the Worker by passing more options to create(): - // https://typescript.temporal.io/api/classes/worker.Worker - // If you need to configure server connection parameters, see docs: - // https://docs.temporal.io/typescript/security#encryption-in-transit-with-mtls + maxConcurrentWorkflowTaskExecutions: 50, + taskQueue: SYNC_TASK_QUEUE + }; + + const webhookWorker = { + connection, + namespace: this.namespace, + workflowsPath: createRequire(import.meta.url).resolve('./workflows'), + activities, + maxConcurrentWorkflowTaskExecutions: 50, + maxActivitiesPerSecond: 50, + taskQueue: WEBHOOK_TASK_QUEUE + }; - await this.worker.run(); + this.workers = await Promise.all([Worker.create(syncWorker), Worker.create(webhookWorker)]); + await Promise.all(this.workers.map((worker) => worker.run())); } stop() { - if (this.worker) { + if (this.workers) { console.log('Stopping Temporal worker'); - this.worker.shutdown(); + this.workers.forEach((worker) => worker.shutdown()); } } } diff --git a/packages/jobs/lib/workflows.ts b/packages/jobs/lib/workflows.ts index 58e7cbaf3f7..c6790edc07d 100644 --- a/packages/jobs/lib/workflows.ts +++ b/packages/jobs/lib/workflows.ts @@ -1,11 +1,11 @@ import { proxyActivities } from '@temporalio/workflow'; import type * as activities from './activities.js'; -import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/worker'; +import type { WebhookArgs, ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/worker'; const DEFAULT_TIMEOUT = '24 hours'; const MAXIMUM_ATTEMPTS = 3; -const { reportFailure, routeSync, scheduleAndRouteSync, runAction } = proxyActivities({ +const { reportFailure, routeSync, scheduleAndRouteSync, runAction, runWebhook } = proxyActivities({ startToCloseTimeout: DEFAULT_TIMEOUT, scheduleToCloseTimeout: DEFAULT_TIMEOUT, retry: { @@ -44,3 +44,13 @@ export async function action(args: ActionArgs): Promise { return { success: false }; } } + +export async function webhook(args: WebhookArgs): Promise { + try { + return await runWebhook(args); + } catch (e: any) { + await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); + + return false; + } +} diff --git a/packages/node-client/lib/index.ts b/packages/node-client/lib/index.ts index eebce667427..107967120d0 100644 --- a/packages/node-client/lib/index.ts +++ b/packages/node-client/lib/index.ts @@ -296,6 +296,28 @@ export class Nango { return axios.post(url, metadata, { headers: this.enrichHeaders(headers) }); } + public async updateMetadata(providerConfigKey: string, connectionId: string, metadata: Record): Promise> { + if (!providerConfigKey) { + throw new Error('Provider Config Key is required'); + } + + if (!connectionId) { + throw new Error('Connection Id is required'); + } + + if (!metadata) { + throw new Error('Metadata is required'); + } + + const url = `${this.serverUrl}/connection/${connectionId}/metadata?provider_config_key=${providerConfigKey}`; + + const headers: Record = { + 'Provider-Config-Key': providerConfigKey as string + }; + + return axios.patch(url, metadata, { headers: this.enrichHeaders(headers) }); + } + public async deleteConnection(providerConfigKey: string, connectionId: string): Promise> { const url = `${this.serverUrl}/connection/${connectionId}?provider_config_key=${providerConfigKey}`; diff --git a/packages/runner/lib/client.unit.test.ts b/packages/runner/lib/client.unit.test.ts index a9c85460451..a5c86775ad1 100644 --- a/packages/runner/lib/client.unit.test.ts +++ b/packages/runner/lib/client.unit.test.ts @@ -36,8 +36,9 @@ describe('Runner client', () => { stubbedMetadata: {} }; const jsCode = 'f = () => { return [1, 2, 3] }; exports.default = f'; - const isAction = true; - const run = client.run.mutate({ nangoProps, isAction, code: jsCode }); + const isInvokedImmediately = true; + const isWebhook = false; + const run = client.run.mutate({ nangoProps, isInvokedImmediately, isWebhook, code: jsCode }); await expect(run).resolves.toEqual([1, 2, 3]); }); }); diff --git a/packages/runner/lib/exec.ts b/packages/runner/lib/exec.ts index add75a6a6b0..78e37471912 100644 --- a/packages/runner/lib/exec.ts +++ b/packages/runner/lib/exec.ts @@ -1,17 +1,18 @@ import type { NangoProps } from '@nangohq/shared'; -import { NangoSync } from '@nangohq/shared'; +import { NangoSync, NangoAction } from '@nangohq/shared'; import * as vm from 'vm'; import * as url from 'url'; import * as crypto from 'crypto'; -export async function exec(nangoProps: NangoProps, isAction: boolean, code: string, codeParams?: object): Promise { - const nangoSync = new NangoSync(nangoProps); +export async function exec(nangoProps: NangoProps, isInvokedImmediately: boolean, isWebhook: boolean, code: string, codeParams?: object): Promise { + const isAction = isInvokedImmediately && !isWebhook; + const nango = isAction ? new NangoAction(nangoProps) : new NangoSync(nangoProps); const wrappedCode = ` (function() { var module = { exports: {} }; var exports = module.exports; ${code} - return module.exports.default || module.exports; + return module.exports; })(); `; @@ -31,14 +32,24 @@ export async function exec(nangoProps: NangoProps, isAction: boolean, code: stri } }; const context = vm.createContext(sandbox); - const f = script.runInContext(context); - if (!f || typeof f !== 'function') { - throw new Error(`Default exports is not a function but a ${typeof f}`); - } - if (isAction) { - return await f(nangoSync, codeParams); + const scriptExports = script.runInContext(context); + if (isWebhook) { + if (!scriptExports.onWebhookPayloadReceived) { + const content = `There is no onWebhookPayloadReceived export for ${nangoProps.syncId}`; + + throw new Error(content); + } + + return await scriptExports.onWebhookPayloadReceived(nango, codeParams); } else { - return await f(nangoSync); + if (!scriptExports.default || typeof scriptExports.default !== 'function') { + throw new Error(`Default exports is not a function but a ${typeof scriptExports.default}`); + } + if (isAction) { + return await scriptExports.default(nango, codeParams); + } else { + return await scriptExports.default(nango); + } } } catch (error) { throw new Error(`Error executing code '${error}'`); diff --git a/packages/runner/lib/server.ts b/packages/runner/lib/server.ts index 6330a41e878..eb529fd80a2 100644 --- a/packages/runner/lib/server.ts +++ b/packages/runner/lib/server.ts @@ -17,7 +17,8 @@ const publicProcedure = t.procedure; //.use(logging); interface RunParams { nangoProps: NangoProps; - isAction: boolean; + isInvokedImmediately: boolean; + isWebhook: boolean; code: string; codeParams?: object; } @@ -44,6 +45,6 @@ function runProcedure() { .input((input) => input as RunParams) .mutation(async ({ input }) => { const { nangoProps, code, codeParams } = input; - return await exec(nangoProps, input.isAction, code, codeParams); + return await exec(nangoProps, input.isInvokedImmediately, input.isWebhook, code, codeParams); }); } diff --git a/packages/server/lib/controllers/apiAuth.controller.ts b/packages/server/lib/controllers/apiAuth.controller.ts index 4d1dfcaf24b..3ca88637af6 100644 --- a/packages/server/lib/controllers/apiAuth.controller.ts +++ b/packages/server/lib/controllers/apiAuth.controller.ts @@ -7,7 +7,7 @@ import { errorManager, analytics, AnalyticsTypes, - SyncClient, + connectionCreated as connectionCreatedHook, createActivityLogMessage, updateSuccess as updateSuccessActivityLog, updateProvider as updateProviderActivityLog, @@ -156,8 +156,15 @@ class ApiAuthController { ); if (updatedConnection) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id: connectionId, + provider_config_key: providerConfigKey, + environment_id: environmentId + }, + config?.provider as string + ); } res.status(200).send({ providerConfigKey: providerConfigKey as string, connectionId: connectionId as string }); @@ -313,8 +320,15 @@ class ApiAuthController { ); if (updatedConnection) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id: connectionId, + provider_config_key: providerConfigKey, + environment_id: environmentId + }, + config?.provider as string + ); } res.status(200).send({ providerConfigKey: providerConfigKey as string, connectionId: connectionId as string }); diff --git a/packages/server/lib/controllers/appAuth.controller.ts b/packages/server/lib/controllers/appAuth.controller.ts index a3bff4ddd50..c91eab8359f 100644 --- a/packages/server/lib/controllers/appAuth.controller.ts +++ b/packages/server/lib/controllers/appAuth.controller.ts @@ -3,7 +3,7 @@ import { environmentService, AuthCredentials, NangoError, - SyncClient, + connectionCreated as connectionCreatedHook, findActivityLogBySession, errorManager, analytics, @@ -160,8 +160,15 @@ class AppAuthController { ); if (updatedConnection) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id: connectionId, + provider_config_key: providerConfigKey, + environment_id: environmentId + }, + session.provider + ); } await createActivityLogMessageAndEnd({ diff --git a/packages/server/lib/controllers/config.controller.integration.test.ts b/packages/server/lib/controllers/config.controller.integration.test.ts index 2a043148f0b..85411f5da88 100644 --- a/packages/server/lib/controllers/config.controller.integration.test.ts +++ b/packages/server/lib/controllers/config.controller.integration.test.ts @@ -217,6 +217,7 @@ describe('Should verify the config controller HTTP API calls', async () => { unique_key: 'test', client_id: 'abc', client_secret: 'def', + has_webhook: false, scopes: 'abc,def,efg', app_link: null, auth_mode: 'OAUTH2', diff --git a/packages/server/lib/controllers/config.controller.ts b/packages/server/lib/controllers/config.controller.ts index f6a2b5bc38d..586e3e9aa61 100644 --- a/packages/server/lib/controllers/config.controller.ts +++ b/packages/server/lib/controllers/config.controller.ts @@ -192,6 +192,7 @@ class ConfigController { }; }); const actions = await getActionsByProviderConfigKey(environmentId, providerConfigKey); + const hasWebhook = providerTemplate.webhook_routing_script; const configRes: ProviderIntegration | IntegrationWithCreds = includeCreds ? ({ @@ -203,7 +204,8 @@ class ConfigController { app_link: config.app_link, auth_mode: authMode, syncs, - actions + actions, + has_webhook: Boolean(hasWebhook) } as IntegrationWithCreds) : ({ unique_key: config.unique_key, provider: config.provider, syncs, actions } as ProviderIntegration); diff --git a/packages/server/lib/controllers/connection.controller.ts b/packages/server/lib/controllers/connection.controller.ts index 69ff56a0026..4f567cb6c06 100644 --- a/packages/server/lib/controllers/connection.controller.ts +++ b/packages/server/lib/controllers/connection.controller.ts @@ -28,8 +28,7 @@ import { createActivityLogAndLogMessage, environmentService, accountService, - SyncClient, - Connection, + connectionCreated as connectionCreatedHook, slackNotificationService } from '@nangohq/shared'; import { getUserAccountAndEnvironmentFromSession } from '../utils/utils.js'; @@ -495,7 +494,7 @@ class ConnectionController { return; } - await connectionService.updateMetadata(connection, req.body); + await connectionService.replaceMetadata(connection, req.body); res.status(201).send(); } catch (err) { @@ -503,6 +502,36 @@ class ConnectionController { } } + async updateMetadata(req: Request, res: Response, next: NextFunction) { + try { + const environmentId = getEnvironmentId(res); + const connectionId = (req.params['connectionId'] as string) || (req.get('Connection-Id') as string); + const providerConfigKey = (req.params['provider_config_key'] as string) || (req.get('Provider-Config-Key') as string); + + const { success, error, response: connection } = await connectionService.getConnection(connectionId, providerConfigKey, environmentId); + + if (!success) { + errorManager.errResFromNangoErr(res, error); + + return; + } + + if (!connection) { + const environmentName = await environmentService.getEnvironmentName(environmentId); + const error = new NangoError('unknown_connection', { connectionId, providerConfigKey, environmentName }); + errorManager.errResFromNangoErr(res, error); + + return; + } + + const metadata = await connectionService.updateMetadata(connection, req.body); + + res.status(200).send(metadata); + } catch (err) { + next(err); + } + } + async createConnection(req: Request, res: Response, next: NextFunction) { try { const environmentId = getEnvironmentId(res); @@ -534,7 +563,7 @@ class ConnectionController { const template = await configService.getTemplate(provider as string); let oAuthCredentials: ImportedCredentials; - let updatedConnection: Connection; + let updatedConnection: { id: number } = {} as { id: number }; if (template.auth_mode === ProviderAuthModes.OAuth2) { const { access_token, refresh_token, expires_at, expires_in, metadata, connection_config } = req.body; @@ -559,7 +588,7 @@ class ConnectionController { raw: req.body.raw || req.body }; - [updatedConnection] = await connectionService.importOAuthConnection( + const [imported] = await connectionService.importOAuthConnection( connection_id, provider_config_key, provider, @@ -567,6 +596,10 @@ class ConnectionController { accountId, oAuthCredentials ); + + if (imported) { + updatedConnection = imported; + } } else if (template.auth_mode === ProviderAuthModes.OAuth1) { const { oauth_token, oauth_token_secret } = req.body; @@ -587,7 +620,7 @@ class ConnectionController { raw: req.body.raw || req.body }; - [updatedConnection] = await connectionService.importOAuthConnection( + const [imported] = await connectionService.importOAuthConnection( connection_id, provider_config_key, provider, @@ -595,6 +628,10 @@ class ConnectionController { accountId, oAuthCredentials ); + + if (imported) { + updatedConnection = imported; + } } else if (template.auth_mode === ProviderAuthModes.Basic) { const { username, password } = req.body; @@ -614,7 +651,7 @@ class ConnectionController { password }; - [updatedConnection] = await connectionService.importApiAuthConnection( + const [imported] = await connectionService.importApiAuthConnection( connection_id, provider_config_key, provider, @@ -622,6 +659,10 @@ class ConnectionController { accountId, credentials ); + + if (imported) { + updatedConnection = imported; + } } else if (template.auth_mode === ProviderAuthModes.ApiKey) { const { api_key: apiKey } = req.body; @@ -635,7 +676,7 @@ class ConnectionController { apiKey }; - [updatedConnection] = await connectionService.importApiAuthConnection( + const [imported] = await connectionService.importApiAuthConnection( connection_id, provider_config_key, provider, @@ -643,6 +684,10 @@ class ConnectionController { accountId, credentials ); + + if (imported) { + updatedConnection = imported; + } } else if (template.auth_mode === ProviderAuthModes.App) { const { app_id, installation_id } = req.body; @@ -675,7 +720,7 @@ class ConnectionController { return; } - [updatedConnection] = await connectionService.upsertConnection( + const [imported] = await connectionService.upsertConnection( connection_id, provider_config_key, provider, @@ -684,14 +729,25 @@ class ConnectionController { environmentId, accountId ); + + if (imported) { + updatedConnection = imported; + } } else { errorManager.errRes(res, 'unknown_oauth_type'); return; } if (updatedConnection && updatedConnection.id) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id, + provider_config_key, + environment_id: environmentId + }, + provider + ); } res.status(201).send(req.body); diff --git a/packages/server/lib/controllers/environment.controller.ts b/packages/server/lib/controllers/environment.controller.ts index 7004934aa2f..67e99f57acc 100644 --- a/packages/server/lib/controllers/environment.controller.ts +++ b/packages/server/lib/controllers/environment.controller.ts @@ -8,6 +8,7 @@ import { isCloud, getWebsocketsPath, getOauthCallbackUrl, + getGlobalWebhookReceiveUrl, getEnvironmentId } from '@nangohq/shared'; import { packageJsonFile, getUserAccountAndEnvironmentFromSession } from '../utils/utils.js'; @@ -53,6 +54,8 @@ class EnvironmentController { } environment.callback_url = await getOauthCallbackUrl(environment.id); + const webhookBaseUrl = await getGlobalWebhookReceiveUrl(); + environment.webhook_receive_url = `${webhookBaseUrl}/${environment.uuid}`; const environmentVariables = await environmentService.getEnvironmentVariables(environment.id); diff --git a/packages/server/lib/controllers/oauth.controller.ts b/packages/server/lib/controllers/oauth.controller.ts index e1f54e62a1c..01406ee2aa1 100644 --- a/packages/server/lib/controllers/oauth.controller.ts +++ b/packages/server/lib/controllers/oauth.controller.ts @@ -11,7 +11,7 @@ import { } from '../utils/utils.js'; import { getConnectionConfig, - SyncClient, + connectionCreated as connectionCreatedHook, interpolateStringFromObject, getOauthCallbackUrl, getGlobalAppCallbackUrl, @@ -910,8 +910,15 @@ class OAuthController { }); if (updatedConnection) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id: connectionId, + provider_config_key: providerConfigKey, + environment_id + }, + session.provider + ); } await updateSuccessActivityLog(activityLogId, true); diff --git a/packages/server/lib/controllers/unauth.controller.ts b/packages/server/lib/controllers/unauth.controller.ts index 12c1f2a934b..c9514498e43 100644 --- a/packages/server/lib/controllers/unauth.controller.ts +++ b/packages/server/lib/controllers/unauth.controller.ts @@ -7,7 +7,7 @@ import { errorManager, analytics, AnalyticsTypes, - SyncClient, + connectionCreated as connectionCreatedHook, createActivityLogMessage, updateSuccess as updateSuccessActivityLog, updateProvider as updateProviderActivityLog, @@ -141,8 +141,15 @@ class UnAuthController { ); if (updatedConnection) { - const syncClient = await SyncClient.getInstance(); - await syncClient?.initiate(updatedConnection.id); + await connectionCreatedHook( + { + id: updatedConnection.id, + connection_id: connectionId, + provider_config_key: providerConfigKey, + environment_id: environmentId + }, + config?.provider as string + ); } res.status(200).send({ providerConfigKey: providerConfigKey as string, connectionId: connectionId as string }); diff --git a/packages/server/lib/controllers/webhook.controller.ts b/packages/server/lib/controllers/webhook.controller.ts new file mode 100644 index 00000000000..f9a4ab69332 --- /dev/null +++ b/packages/server/lib/controllers/webhook.controller.ts @@ -0,0 +1,32 @@ +import type { Request, Response, NextFunction } from 'express'; +import { routeWebhook, featureFlags, environmentService } from '@nangohq/shared'; + +class WebhookController { + async receive(req: Request, res: Response, next: NextFunction) { + const { environmentUuid, providerConfigKey } = req.params; + const headers = req.headers; + try { + if (!environmentUuid || !providerConfigKey) { + return; + } + + const accountUUID = await environmentService.getAccountUUIDFromEnvironmentUUID(environmentUuid); + + if (!accountUUID) { + res.status(404).send(); + return; + } + + const areWebhooksEnabled = await featureFlags.isEnabled('external-webhooks', accountUUID, true); + + if (areWebhooksEnabled) { + routeWebhook(environmentUuid, providerConfigKey, headers, req.body); + } + res.status(200).send(); + } catch (err) { + next(err); + } + } +} + +export default new WebhookController(); diff --git a/packages/server/lib/server.ts b/packages/server/lib/server.ts index 549c2e29f5f..fdc9e04111e 100644 --- a/packages/server/lib/server.ts +++ b/packages/server/lib/server.ts @@ -20,6 +20,7 @@ import flowController from './controllers/flow.controller.js'; import apiAuthController from './controllers/apiAuth.controller.js'; import appAuthController from './controllers/appAuth.controller.js'; import onboardingController from './controllers/onboarding.controller.js'; +import webhookController from './controllers/webhook.controller.js'; import path from 'path'; import { packageJsonFile, dirname } from './utils/utils.js'; import { WebSocketServer, WebSocket } from 'ws'; @@ -75,6 +76,7 @@ app.get('/health', (_, res) => { app.route('/oauth/callback').get(oauthController.oauthCallback.bind(oauthController)); app.route('/app-auth/connect').get(appAuthController.connect.bind(appAuthController)); app.route('/oauth/connect/:providerConfigKey').get(apiPublicAuth, oauthController.oauthRequest.bind(oauthController)); +app.route('/webhook/:environmentUuid/:providerConfigKey').post(webhookController.receive.bind(proxyController)); app.route('/api-auth/api-key/:providerConfigKey').post(apiPublicAuth, apiAuthController.apiKey.bind(authController)); app.route('/api-auth/basic/:providerConfigKey').post(apiPublicAuth, apiAuthController.basic.bind(authController)); app.route('/unauth/:providerConfigKey').post(apiPublicAuth, unAuthController.create.bind(unAuthController)); @@ -91,6 +93,7 @@ app.route('/connection/:connectionId').get(apiAuth, connectionController.getConn app.route('/connection').get(apiAuth, connectionController.listConnections.bind(connectionController)); app.route('/connection/:connectionId').delete(apiAuth, connectionController.deleteConnection.bind(connectionController)); app.route('/connection/:connectionId/metadata').post(apiAuth, connectionController.setMetadata.bind(connectionController)); +app.route('/connection/:connectionId/metadata').patch(apiAuth, connectionController.updateMetadata.bind(connectionController)); app.route('/connection').post(apiAuth, connectionController.createConnection.bind(connectionController)); app.route('/environment-variables').get(apiAuth, environmentController.getEnvironmentVariables.bind(connectionController)); app.route('/sync/deploy').post(apiAuth, syncController.deploySync.bind(syncController)); diff --git a/packages/shared/lib/clients/sync.client.ts b/packages/shared/lib/clients/sync.client.ts index 24b109fb1d5..dff29d3f5cb 100644 --- a/packages/shared/lib/clients/sync.client.ts +++ b/packages/shared/lib/clients/sync.client.ts @@ -1,5 +1,5 @@ import { Client, Connection, ScheduleOverlapPolicy, ScheduleDescription } from '@temporalio/client'; -import type { NangoConnection } from '../models/Connection.js'; +import type { NangoConnection, Connection as NangoFullConnection } from '../models/Connection.js'; import ms from 'ms'; import fs from 'fs-extra'; import type { Config as ProviderConfig } from '../models/Provider.js'; @@ -7,7 +7,7 @@ import type { NangoIntegrationData, NangoConfig, NangoIntegration } from '../mod import { Sync, SyncStatus, SyncType, ScheduleStatus, SyncCommand, SyncWithSchedule } from '../models/Sync.js'; import type { ServiceResponse } from '../models/Generic.js'; import { LogActionEnum, LogLevel } from '../models/Activity.js'; -import { TASK_QUEUE } from '../constants.js'; +import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE } from '../constants.js'; import { createActivityLog, createActivityLogMessage, @@ -25,9 +25,11 @@ import errorManager, { ErrorSourceEnum } from '../utils/error.manager.js'; import { NangoError } from '../utils/error.js'; import { isProd } from '../utils/utils.js'; -const generateActionWorkflowId = (actionName: string, connectionId: string) => `${TASK_QUEUE}.ACTION:${actionName}.${connectionId}.${Date.now()}`; -const generateWorkflowId = (sync: Sync, syncName: string, connectionId: string) => `${TASK_QUEUE}.${syncName}.${connectionId}-${sync.id}`; -const generateScheduleId = (sync: Sync, syncName: string, connectionId: string) => `${TASK_QUEUE}.${syncName}.${connectionId}-schedule-${sync.id}`; +const generateActionWorkflowId = (actionName: string, connectionId: string) => `${SYNC_TASK_QUEUE}.ACTION:${actionName}.${connectionId}.${Date.now()}`; +const generateWebhookWorkflowId = (parentSyncName: string, webhookName: string, connectionId: string) => + `${WEBHOOK_TASK_QUEUE}.WEBHOOK:${parentSyncName}:${webhookName}.${connectionId}.${Date.now()}`; +const generateWorkflowId = (sync: Sync, syncName: string, connectionId: string) => `${SYNC_TASK_QUEUE}.${syncName}.${connectionId}-${sync.id}`; +const generateScheduleId = (sync: Sync, syncName: string, connectionId: string) => `${SYNC_TASK_QUEUE}.${syncName}.${connectionId}-schedule-${sync.id}`; const OVERLAP_POLICY: ScheduleOverlapPolicy = ScheduleOverlapPolicy.BUFFER_ONE; @@ -207,7 +209,7 @@ class SyncClient { } handle = await this.client?.workflow.start('initialSync', { - taskQueue: TASK_QUEUE, + taskQueue: SYNC_TASK_QUEUE, workflowId: jobId, args: [ { @@ -247,7 +249,7 @@ class SyncClient { action: { type: 'startWorkflow', workflowType: 'continuousSync', - taskQueue: TASK_QUEUE, + taskQueue: SYNC_TASK_QUEUE, args: [ { syncId: sync.id, @@ -277,7 +279,7 @@ class SyncClient { level: 'info', environment_id: nangoConnection?.environment_id as number, activity_log_id: activityLogId as number, - content: `Started initial background sync ${handle?.workflowId} and data updated on a schedule ${scheduleId} at ${syncData.runs} in the task queue: ${TASK_QUEUE}`, + content: `Started initial background sync ${handle?.workflowId} and data updated on a schedule ${scheduleId} at ${syncData.runs} in the task queue: ${SYNC_TASK_QUEUE}`, timestamp: Date.now() }); } @@ -459,7 +461,7 @@ class SyncClient { level: 'info', environment_id, activity_log_id: activityLogId as number, - content: `Starting action workflow ${workflowId} in the task queue: ${TASK_QUEUE}`, + content: `Starting action workflow ${workflowId} in the task queue: ${SYNC_TASK_QUEUE}`, params: { input: JSON.stringify(input, null, 2) }, @@ -468,7 +470,7 @@ class SyncClient { } const actionHandler = await this.client?.workflow.execute('action', { - taskQueue: TASK_QUEUE, + taskQueue: SYNC_TASK_QUEUE, workflowId, args: [ { @@ -539,6 +541,114 @@ class SyncClient { } } + async triggerWebhook( + nangoConnection: NangoConnection, + webhookName: string, + provider: string, + parentSyncName: string, + input: object, + environment_id: number + ): Promise> { + const log = { + level: 'info' as LogLevel, + success: null, + action: LogActionEnum.WEBHOOK, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: nangoConnection?.connection_id as string, + provider_config_key: nangoConnection?.provider_config_key as string, + provider, + environment_id: nangoConnection?.environment_id as number, + operation_name: webhookName + }; + + const activityLogId = await createActivityLog(log); + + const workflowId = generateWebhookWorkflowId(parentSyncName, webhookName, nangoConnection.connection_id as string); + + try { + await createActivityLogMessage({ + level: 'info', + environment_id, + activity_log_id: activityLogId as number, + content: `Starting webhook workflow ${workflowId} in the task queue: ${WEBHOOK_TASK_QUEUE}`, + params: { + input: JSON.stringify(input, null, 2) + }, + timestamp: Date.now() + }); + + const { credentials, credentials_iv, credentials_tag, deleted, deleted_at, ...nangoConnectionWithoutCredentials } = + nangoConnection as unknown as NangoFullConnection; + + const webhookHandler = await this.client?.workflow.execute('webhook', { + taskQueue: WEBHOOK_TASK_QUEUE, + workflowId, + args: [ + { + name: webhookName, + parentSyncName, + nangoConnection: nangoConnectionWithoutCredentials, + input, + activityLogId + } + ] + }); + + const { success, error, response } = webhookHandler; + + if (success === false || error) { + await createActivityLogMessageAndEnd({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `The webhook workflow ${workflowId} did not complete successfully` + }); + + return { success, error, response }; + } + + await createActivityLogMessageAndEnd({ + level: 'info', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `The webhook workflow ${workflowId} was successfully run.` + }); + + await updateSuccessActivityLog(activityLogId as number, true); + + return { success, error, response }; + } catch (e) { + const errorMessage = JSON.stringify(e, ['message', 'name'], 2); + const error = new NangoError('webhook_script_failure', { errorMessage }); + + await createActivityLogMessageAndEnd({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `The webhook workflow ${workflowId} failed with error: ${e}` + }); + + await errorManager.report(e, { + source: ErrorSourceEnum.PLATFORM, + operation: LogActionEnum.SYNC_CLIENT, + environmentId: nangoConnection.environment_id, + metadata: { + parentSyncName, + webhookName, + connectionDetails: JSON.stringify(nangoConnection), + input + } + }); + + return { success: false, error, response: null }; + } + } + async updateSyncSchedule(schedule_id: string, interval: string, offset: number, environmentId: number, syncName?: string, activityLogId?: number) { function updateFunction(scheduleDescription: ScheduleDescription) { scheduleDescription.spec = { diff --git a/packages/shared/lib/constants.ts b/packages/shared/lib/constants.ts index fef07f0694c..206a1ef7a09 100644 --- a/packages/shared/lib/constants.ts +++ b/packages/shared/lib/constants.ts @@ -1 +1,2 @@ -export const TASK_QUEUE = 'nango-syncs'; +export const SYNC_TASK_QUEUE = 'nango-syncs'; +export const WEBHOOK_TASK_QUEUE = 'nango-webhooks'; diff --git a/packages/shared/lib/db/migrations/20231204191116_add_webhooks_subscriptions_to_config.cjs b/packages/shared/lib/db/migrations/20231204191116_add_webhooks_subscriptions_to_config.cjs new file mode 100644 index 00000000000..e982febaac6 --- /dev/null +++ b/packages/shared/lib/db/migrations/20231204191116_add_webhooks_subscriptions_to_config.cjs @@ -0,0 +1,13 @@ +const TABLE_NAME = '_nango_sync_configs'; + +exports.up = function (knex, _) { + return knex.schema.withSchema('nango').alterTable(TABLE_NAME, function (table) { + table.specificType('webhook_subscriptions', 'text ARRAY'); + }); +}; + +exports.down = function (knex, _) { + return knex.schema.withSchema('nango').alterTable(TABLE_NAME, function (table) { + table.dropColumn('webhook_subscriptions'); + }); +}; diff --git a/packages/shared/lib/db/migrations/20231205151403_add_uuid_to_environment.cjs b/packages/shared/lib/db/migrations/20231205151403_add_uuid_to_environment.cjs new file mode 100644 index 00000000000..70582c15447 --- /dev/null +++ b/packages/shared/lib/db/migrations/20231205151403_add_uuid_to_environment.cjs @@ -0,0 +1,13 @@ +const tableName = '_nango_environments'; + +exports.up = async function (knex, _) { + await knex.schema.withSchema('nango').alterTable(tableName, function (table) { + table.uuid('uuid').defaultTo(knex.raw('uuid_generate_v4()')).index(); + }); +}; + +exports.down = async function (knex, _) { + await knex.schema.withSchema('nango').table(tableName, function (table) { + table.dropColumn('uuid'); + }); +}; diff --git a/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs new file mode 100644 index 00000000000..53e22968e46 --- /dev/null +++ b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs @@ -0,0 +1,31 @@ +const tableName = '_nango_sync_jobs'; + +exports.up = function(knex, _) { + return knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.string('type_new').defaultsTo('initial').notNullable(); + }) + .then(() => knex.raw(` + UPDATE nango.${tableName} SET type_new = type; + `)) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.dropColumn('type'); + })) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.renameColumn('type_new', 'type'); + })); +}; + +exports.down = function(knex, _) { + return knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.enu('type', ['INITIAL', 'INCREMENTAL']).defaultTo('initial').notNullable(); + }) + .then(() => knex.raw(` + UPDATE nango.${tableName} SET type_new = type; + `)) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.dropColumn('type'); + })) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.renameColumn('type_new', 'type'); + })); +}; diff --git a/packages/shared/lib/hooks/hooks.ts b/packages/shared/lib/hooks/hooks.ts new file mode 100644 index 00000000000..a4f9ecc6877 --- /dev/null +++ b/packages/shared/lib/hooks/hooks.ts @@ -0,0 +1,10 @@ +import SyncClient from '../clients/sync.client.js'; +import type { RecentlyCreatedConnection } from '../models/Connection.js'; +import integrationPostConnectionScript from '../integrations/scripts/connection/connection.manager.js'; + +export const connectionCreated = async (connection: RecentlyCreatedConnection, provider: string): Promise => { + const syncClient = await SyncClient.getInstance(); + syncClient?.initiate(connection.id as number); + + integrationPostConnectionScript(connection, provider); +}; diff --git a/packages/shared/lib/index.ts b/packages/shared/lib/index.ts index e74bca4682c..f6f225ba1d3 100644 --- a/packages/shared/lib/index.ts +++ b/packages/shared/lib/index.ts @@ -20,6 +20,7 @@ import flowService from './services/flow.service.js'; import slackNotificationService from './services/sync/notification/slack.service.js'; import analytics, { AnalyticsTypes } from './utils/analytics.js'; import logger from './logger/console.js'; +import routeWebhook from './integrations/scripts/webhook/webhook.manager.js'; import featureFlags from './utils/featureflags.js'; export * from './services/activity/activity.service.js'; @@ -31,6 +32,8 @@ export * from './services/sync/config/endpoint.service.js'; export * from './services/sync/config/deploy.service.js'; export * from './services/onboarding.service.js'; +export * from './hooks/hooks.js'; + export * as dataService from './services/sync/data/data.service.js'; export * as syncDataService from './services/sync/data/records.service.js'; @@ -72,6 +75,7 @@ export { slackNotificationService, analytics, AnalyticsTypes, + routeWebhook, logger, featureFlags }; diff --git a/packages/shared/lib/integrations/scripts/connection/connection.manager.ts b/packages/shared/lib/integrations/scripts/connection/connection.manager.ts new file mode 100644 index 00000000000..935d2706ce6 --- /dev/null +++ b/packages/shared/lib/integrations/scripts/connection/connection.manager.ts @@ -0,0 +1,131 @@ +import type { AxiosResponse } from 'axios'; +import type { RecentlyCreatedConnection, Connection, ConnectionConfig } from '../../../models/Connection.js'; +import { LogLevel, LogActionEnum } from '../../../models/Activity.js'; +import { createActivityLogAndLogMessage } from '../../../services/activity/activity.service.js'; +import type { HTTP_VERB } from '../../../models/Generic.js'; +import proxyService from '../../../services/proxy.service.js'; +import connectionService from '../../../services/connection.service.js'; +import environmentService from '../../../services/environment.service.js'; +import metricsManager, { MetricTypes } from '../../../utils/metrics.manager.js'; + +import * as postConnectionHandlers from './index.js'; + +interface PostConnectionHandler { + (internalNango: InternalNango): Promise; +} + +type PostConnectionHandlersMap = { [key: string]: PostConnectionHandler }; + +const handlers: PostConnectionHandlersMap = postConnectionHandlers as unknown as PostConnectionHandlersMap; + +export interface InternalNango { + proxy: ({ method, endpoint, data }: { method?: HTTP_VERB; endpoint: string; data?: unknown }) => Promise; + updateConnectionConfig: (config: ConnectionConfig) => Promise; +} + +async function execute(createdConnection: RecentlyCreatedConnection, provider: string) { + const { connection_id, environment_id, provider_config_key } = createdConnection; + try { + const accountId = await environmentService.getAccountIdFromEnvironment(environment_id); + const { success, response: connection } = await connectionService.getConnectionCredentials( + accountId as number, + environment_id, + connection_id, + provider_config_key + ); + + if (!success || !connection) { + return; + } + + const internalConfig = { + environmentId: createdConnection.environment_id, + isFlow: true, + isDryRun: false, + throwErrors: false, + connection + }; + + const externalConfig = { + endpoint: '', + connectionId: connection.connection_id, + providerConfigKey: connection.provider_config_key, + method: 'GET' as HTTP_VERB, + data: {} + }; + + const internalNango: InternalNango = { + proxy: ({ method, endpoint, data }: { endpoint: string; method?: HTTP_VERB; data?: unknown }) => { + const finalExternalConfig = { ...externalConfig, method: method || externalConfig.method, endpoint }; + if (data) { + finalExternalConfig.data = data; + } + return proxyService.routeOrConfigure(finalExternalConfig, internalConfig) as Promise; + }, + updateConnectionConfig: (connectionConfig: ConnectionConfig) => { + return connectionService.updateConnectionConfig(connection as unknown as Connection, connectionConfig); + } + }; + + const handler = handlers[`${provider}PostConnection`]; + + if (handler) { + try { + await handler(internalNango); + } catch (e: any) { + const errorMessage = e.message || 'Unknown error'; + const errorDetails = { + message: errorMessage, + name: e.name || 'Error', + stack: e.stack || 'No stack trace' + }; + + const errorString = JSON.stringify(errorDetails); + const log = { + level: 'error' as LogLevel, + success: false, + action: LogActionEnum.AUTH, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: connection_id, + provider: '', + provider_config_key: provider_config_key, + environment_id + }; + + await createActivityLogAndLogMessage(log, { + level: 'error', + environment_id: environment_id, + timestamp: Date.now(), + content: `Post connection script failed with the error: ${errorString}` + }); + + await metricsManager.capture(MetricTypes.POST_CONNECTION_SCRIPT_FAILURE, `Post connection script failed, ${errorString}`, LogActionEnum.AUTH, { + environmentId: String(environment_id), + connectionId: connection_id, + providerConfigKey: provider_config_key, + provider: provider + }); + } + } + } catch (e: any) { + const errorMessage = e.message || 'Unknown error'; + const errorDetails = { + message: errorMessage, + name: e.name || 'Error', + stack: e.stack || 'No stack trace' + }; + + const errorString = JSON.stringify(errorDetails); + + await metricsManager.capture(MetricTypes.POST_CONNECTION_SCRIPT_FAILURE, `Post connection manager failed, ${errorString}`, LogActionEnum.AUTH, { + environmentId: String(environment_id), + connectionId: connection_id, + providerConfigKey: provider_config_key, + provider: provider + }); + } +} + +export default execute; diff --git a/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts new file mode 100644 index 00000000000..23d6bac5d8e --- /dev/null +++ b/packages/shared/lib/integrations/scripts/connection/hubspot-post-connection.ts @@ -0,0 +1,12 @@ +import type { InternalNango as Nango } from './connection.manager.js'; + +export default async function execute(nango: Nango) { + const response = await nango.proxy({ endpoint: '/account-info/v3/details' }); + + if (!response || !response.data || !response.data.portalId) { + return; + } + const portalId = response.data.portalId; + + await nango.updateConnectionConfig({ portalId }); +} diff --git a/packages/shared/lib/integrations/scripts/connection/index.ts b/packages/shared/lib/integrations/scripts/connection/index.ts new file mode 100644 index 00000000000..278fef580e5 --- /dev/null +++ b/packages/shared/lib/integrations/scripts/connection/index.ts @@ -0,0 +1,2 @@ +export { default as hubspotPostConnection } from './hubspot-post-connection.js'; +export { default as jiraPostConnection } from './jira-post-connection.js'; diff --git a/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts b/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts new file mode 100644 index 00000000000..15ac64d1703 --- /dev/null +++ b/packages/shared/lib/integrations/scripts/connection/jira-post-connection.ts @@ -0,0 +1,26 @@ +import type { InternalNango as Nango } from './connection.manager.js'; + +export default async function execute(nango: Nango) { + const response = await nango.proxy({ + endpoint: `oauth/token/accessible-resources` + }); + + if (!response || !response.data || response.data.length === 0 || !response.data[0].id) { + return; + } + + const cloudId = response.data[0].id; + + const accountResponse = await nango.proxy({ + endpoint: `ex/jira/${cloudId}/rest/api/3/myself` + }); + + if (!accountResponse || !accountResponse.data || accountResponse.data.length === 0) { + await nango.updateConnectionConfig({ cloudId }); + return; + } + + const { accountId } = accountResponse.data; + + await nango.updateConnectionConfig({ cloudId, accountId }); +} diff --git a/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts b/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts new file mode 100644 index 00000000000..8dea8921c73 --- /dev/null +++ b/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts @@ -0,0 +1,28 @@ +import type { InternalNango as Nango } from './webhook.manager.js'; +import type { Config as ProviderConfig } from '../../../models/Provider.js'; +import crypto from 'crypto'; + +export function validate(integration: ProviderConfig, headers: Record, body: any): boolean { + const signature = headers['x-hubspot-signature']; + + const combinedSignature = `${integration.oauth_client_secret}${JSON.stringify(body)}`; + const createdHash = crypto.createHash('sha256').update(combinedSignature).digest('hex'); + + return signature === createdHash; +} + +export default async function route(nango: Nango, integration: ProviderConfig, headers: Record, body: any) { + const valid = validate(integration, headers, body); + + if (!valid) { + return; + } + + if (Array.isArray(body)) { + for (const event of body) { + await nango.executeScriptForWebhooks(integration, event, 'subscriptionType', 'portalId'); + } + } else { + await nango.executeScriptForWebhooks(integration, body, 'subscriptionType', 'portalId'); + } +} diff --git a/packages/shared/lib/integrations/scripts/webhook/index.ts b/packages/shared/lib/integrations/scripts/webhook/index.ts new file mode 100644 index 00000000000..7f7dcd431d2 --- /dev/null +++ b/packages/shared/lib/integrations/scripts/webhook/index.ts @@ -0,0 +1,2 @@ +export { default as hubspotWebhook } from './hubspot-webhook-routing.js'; +export { default as jiraWebhook } from './jira-webhook-routing.js'; diff --git a/packages/shared/lib/integrations/scripts/webhook/jira-webhook-routing.ts b/packages/shared/lib/integrations/scripts/webhook/jira-webhook-routing.ts new file mode 100644 index 00000000000..12cd818204b --- /dev/null +++ b/packages/shared/lib/integrations/scripts/webhook/jira-webhook-routing.ts @@ -0,0 +1,12 @@ +import type { InternalNango as Nango } from './webhook.manager.js'; +import type { Config as ProviderConfig } from '../../../models/Provider.js'; + +export default async function route(nango: Nango, integration: ProviderConfig, _headers: Record, body: any) { + if (Array.isArray(body)) { + for (const event of body) { + await nango.executeScriptForWebhooks(integration, event, 'payload.webhookEvent', 'payload.user.accountId', 'accountId'); + } + } else { + await nango.executeScriptForWebhooks(integration, body, 'payload.webhookEvent', 'payload.user.accountId', 'accountId'); + } +} diff --git a/packages/shared/lib/integrations/scripts/webhook/webhook.manager.ts b/packages/shared/lib/integrations/scripts/webhook/webhook.manager.ts new file mode 100644 index 00000000000..d90421518ba --- /dev/null +++ b/packages/shared/lib/integrations/scripts/webhook/webhook.manager.ts @@ -0,0 +1,105 @@ +import get from 'lodash-es/get.js'; +import configService from '../../../services/config.service.js'; +import SyncClient from '../../../clients/sync.client.js'; +import connectionService from '../../../services/connection.service.js'; +import { getSyncConfigsByConfigId } from '../../../services/sync/config/config.service.js'; +import type { SyncConfig } from './../../../models/Sync.js'; +import type { Config as ProviderConfig } from './../../../models/Provider.js'; +import webhookService from '../../../services/sync/notification/webhook.service.js'; +import environmentService from '../../../services/environment.service.js'; +import metricsManager, { MetricTypes } from '../../../utils/metrics.manager.js'; +import { LogActionEnum } from '../../../models/Activity.js'; + +import * as webhookHandlers from './index.js'; + +interface WebhookHandler { + (internalNango: InternalNango, integration: ProviderConfig, headers: Record, body: any): Promise; +} + +type WebhookHandlersMap = { [key: string]: WebhookHandler }; + +const handlers: WebhookHandlersMap = webhookHandlers as unknown as WebhookHandlersMap; + +export interface InternalNango { + getWebhooks: (environment_id: number, nango_config_id: number) => Promise; + executeScriptForWebhooks(integration: ProviderConfig, body: any, webhookType: string, connectionIdentifier: string, propName?: string): Promise; +} + +const internalNango: InternalNango = { + getWebhooks: async (environment_id: number, nango_config_id: number) => { + const syncConfigs = await getSyncConfigsByConfigId(environment_id, nango_config_id); + + if (!syncConfigs) { + return null; + } + + const syncConfigsWithWebhooks = syncConfigs.filter((syncConfig: SyncConfig) => syncConfig.webhook_subscriptions); + + return syncConfigsWithWebhooks; + }, + executeScriptForWebhooks: async (integration: ProviderConfig, body: any, webhookType: string, connectionIdentifier: string, propName?: string) => { + const syncConfigsWithWebhooks = await internalNango.getWebhooks(integration.environment_id, integration.id as number); + + if (!syncConfigsWithWebhooks) { + return; + } + const syncClient = await SyncClient.getInstance(); + + if (!get(body, connectionIdentifier)) { + return; + } + + const connection = await connectionService.findConnectionByConnectionConfigValue(propName || connectionIdentifier, get(body, connectionIdentifier)); + + if (!connection) { + return; + } + + const accountId = await environmentService.getAccountIdFromEnvironment(integration.environment_id); + + await metricsManager.capture(MetricTypes.INCOMING_WEBHOOK_RECEIVED, 'Incoming webhook received and connection found for it', LogActionEnum.WEBHOOK, { + accountId: String(accountId), + environmentId: String(integration.environment_id), + provider: integration.provider, + providerConfigKey: integration.unique_key, + connectionId: String(connection.connection_id) + }); + + for (const syncConfig of syncConfigsWithWebhooks) { + const { webhook_subscriptions } = syncConfig; + + if (!webhook_subscriptions) { + continue; + } + + for (const webhook of webhook_subscriptions) { + if (get(body, webhookType) === webhook) { + await syncClient?.triggerWebhook(connection, integration.provider, webhook, syncConfig.sync_name, body, integration.environment_id); + } + } + } + } +}; + +async function execute(environmentUuid: string, providerConfigKey: string, headers: Record, body: any) { + if (!body) { + return; + } + + const provider = await configService.getProviderName(providerConfigKey); + const integration = await configService.getProviderConfigByUuid(providerConfigKey, environmentUuid); + + if (!provider || !integration) { + return; + } + + const handler = handlers[`${provider}Webhook`]; + + if (handler) { + await handler(internalNango, integration, headers, body); + } + + await webhookService.forward(integration.environment_id, providerConfigKey, provider, body); +} + +export default execute; diff --git a/packages/shared/lib/models/Activity.ts b/packages/shared/lib/models/Activity.ts index ff6efc5cb80..1dfcf95f017 100644 --- a/packages/shared/lib/models/Activity.ts +++ b/packages/shared/lib/models/Activity.ts @@ -17,7 +17,8 @@ export type LogAction = | 'sync client' | 'sync deploy' | 'token' - | 'trigger sync'; + | 'trigger sync' + | 'webhook'; export enum LogActionEnum { ACCOUNT = 'account', @@ -35,7 +36,8 @@ export enum LogActionEnum { SYNC_CLIENT = 'sync client', SYNC_DEPLOY = 'sync deploy', TOKEN = 'token', - TRIGGER_SYNC = 'trigger sync' + TRIGGER_SYNC = 'trigger sync', + WEBHOOK = 'webhook' } interface Message { diff --git a/packages/shared/lib/models/Connection.ts b/packages/shared/lib/models/Connection.ts index a512fcee6cb..028fe6b8883 100644 --- a/packages/shared/lib/models/Connection.ts +++ b/packages/shared/lib/models/Connection.ts @@ -5,11 +5,15 @@ export interface Metadata { [key: string]: string | Record; } +export interface ConnectionConfig { + [key: string]: any; +} + export interface BaseConnection extends TimestampsAndDeleted { id?: number; provider_config_key: string; connection_id: string; - connection_config: Record; + connection_config: ConnectionConfig; environment_id: number; metadata?: Metadata | null; credentials_iv?: string | null; @@ -25,12 +29,14 @@ export interface Connection extends BaseConnection { credentials: AuthCredentials | ApiKeyCredentials | BasicApiCredentials | AppCredentials; } +export type RecentlyCreatedConnection = Pick; + export interface ApiConnection { id?: number; connection_id: string; provider_config_key: string; environment_id: number; - connection_config: Record; + connection_config: ConnectionConfig; credentials_iv?: string | null; credentials_tag?: string | null; credentials: BasicApiCredentials | ApiKeyCredentials; @@ -41,6 +47,7 @@ export interface NangoConnection { connection_id: string; provider_config_key: string; environment_id: number; + connection_config?: ConnectionConfig; // TODO legacy while the migration is in progress account_id?: number; diff --git a/packages/shared/lib/models/Environment.ts b/packages/shared/lib/models/Environment.ts index 86f3c520513..ae18127c74b 100644 --- a/packages/shared/lib/models/Environment.ts +++ b/packages/shared/lib/models/Environment.ts @@ -2,6 +2,7 @@ import type { Timestamps } from './Generic.js'; export interface Environment extends Timestamps { id: number; + uuid?: string; name: string; account_id: number; secret_key: string; @@ -24,4 +25,6 @@ export interface Environment extends Timestamps { pending_secret_key_tag?: string | null; pending_public_key?: string | null; slack_notifications?: boolean; + + webhook_receive_url?: string; } diff --git a/packages/shared/lib/models/NangoConfig.ts b/packages/shared/lib/models/NangoConfig.ts index 09e721debf1..9e5f2148689 100644 --- a/packages/shared/lib/models/NangoConfig.ts +++ b/packages/shared/lib/models/NangoConfig.ts @@ -21,6 +21,7 @@ export interface NangoIntegrationDataV1 { export interface NangoIntegrationDataV2 extends NangoIntegrationDataV1 { sync_type?: SyncType; description?: string; + 'webhook-subscriptions'?: string[]; scopes?: string[]; output?: string | string[]; } @@ -120,6 +121,7 @@ export interface NangoSyncConfig { // v2 additions input?: NangoSyncModel; sync_type?: SyncType; + webhookSubscriptions?: string[]; } export interface StandardNangoConfig { diff --git a/packages/shared/lib/models/Provider.ts b/packages/shared/lib/models/Provider.ts index 82a778b8c39..75e7eae493d 100644 --- a/packages/shared/lib/models/Provider.ts +++ b/packages/shared/lib/models/Provider.ts @@ -43,6 +43,8 @@ export interface Template { token_response_metadata?: Array; docs?: string; token_expiration_buffer?: number; // In seconds. + webhook_routing_script?: string; + post_connection_script?: string; } export interface TemplateAlias { diff --git a/packages/shared/lib/models/Sync.ts b/packages/shared/lib/models/Sync.ts index 6158a37bda0..16b075c7c3a 100644 --- a/packages/shared/lib/models/Sync.ts +++ b/packages/shared/lib/models/Sync.ts @@ -15,6 +15,7 @@ export enum SyncStatus { export enum SyncType { INITIAL = 'INITIAL', INCREMENTAL = 'INCREMENTAL', + WEBHOOK = 'WEBHOOK', FULL = 'FULL', ACTION = 'ACTION' } @@ -102,6 +103,7 @@ export interface SyncConfig extends TimestampsAndDeleted { endpoints?: NangoSyncEndpoint[]; input?: string; sync_type?: SyncType | undefined; + webhook_subscriptions?: string[]; } export interface SyncEndpoint extends Timestamps { @@ -187,6 +189,7 @@ export interface IncomingFlowConfig extends InternalIncomingPreBuiltFlowConfig { track_deletes?: boolean; input?: string; sync_type?: SyncType; + webhookSubscriptions?: string[]; } export enum ScheduleStatus { @@ -268,6 +271,7 @@ export const SyncCommandToScheduleStatus = { }; export interface NangoSyncWebhookBody { + from: string; connectionId: string; providerConfigKey: string; syncName: string; @@ -297,7 +301,8 @@ export interface IntegrationServiceInterface { integrationData: NangoIntegrationData, environmentId: number, writeToDb: boolean, - isAction: boolean, + isInvokedImmediately: boolean, + isWebhook: boolean, optionalLoadLocation?: string, input?: object, temporalContext?: Context diff --git a/packages/shared/lib/sdk/sync.ts b/packages/shared/lib/sdk/sync.ts index f35371cbf5b..85d10804e3b 100644 --- a/packages/shared/lib/sdk/sync.ts +++ b/packages/shared/lib/sdk/sync.ts @@ -1,5 +1,5 @@ import { getSyncConfigByJobId } from '../services/sync/config/config.service.js'; -import { upsert } from '../services/sync/data/data.service.js'; +import { updateRecord, upsert } from '../services/sync/data/data.service.js'; import { formatDataRecords } from '../services/sync/data/records.service.js'; import { createActivityLogMessage } from '../services/activity/activity.service.js'; import { setLastSyncDate } from '../services/sync/sync.service.js'; @@ -331,6 +331,10 @@ export class NangoAction { return this.nango.setMetadata(this.providerConfigKey as string, this.connectionId as string, metadata); } + public async updateMetadata(metadata: Record): Promise> { + return this.nango.updateMetadata(this.providerConfigKey as string, this.connectionId as string, metadata); + } + public async setFieldMapping(fieldMapping: Record): Promise> { console.warn('setFieldMapping is deprecated. Please use setMetadata instead.'); return this.nango.setMetadata(this.providerConfigKey as string, this.connectionId as string, fieldMapping); @@ -498,7 +502,7 @@ export class NangoSync extends NangoAction { public async batchSave(results: T[], model: string): Promise { if (!results || results.length === 0) { if (this.dryRun) { - console.log('batchSave received an empty array. No records to send.'); + console.log('batchSave received an empty array. No records to save.'); } return true; } @@ -724,6 +728,114 @@ export class NangoSync extends NangoAction { throw new Error(responseResults?.error); } } + + public async batchUpdate(results: T[], model: string): Promise { + if (!results || results.length === 0) { + if (this.dryRun) { + console.log('batchUpdate received an empty array. No records to update.'); + } + return true; + } + + if (!this.nangoConnectionId || !this.activityLogId) { + throw new Error('Nango Connection Id, and Activity Log Id both required'); + } + + const { + success, + error, + response: formattedResults + } = formatDataRecords( + results as unknown as DataResponse[], + this.nangoConnectionId as number, + model, + this.syncId as string, + this.syncJobId || 0, + this.lastSyncDate, + false + ); + + if (!success || formattedResults === null) { + if (!this.dryRun) { + await createActivityLogMessage({ + level: 'error', + environment_id: this.environmentId as number, + activity_log_id: this.activityLogId as number, + content: `There was an issue with the batch save. ${error?.message}`, + timestamp: Date.now() + }); + } + + throw error; + } + + if (this.dryRun) { + this.logMessages?.push(`A batch update call would update the following data to the ${model} model:`); + this.logMessages?.push(...results); + return null; + } + + const responseResults = await updateRecord( + formattedResults, + '_nango_sync_data_records', + 'external_id', + this.nangoConnectionId as number, + model, + this.activityLogId as number, + this.environmentId as number + ); + + if (responseResults.success) { + const { summary } = responseResults; + const updatedResults = { + [model]: { + added: summary?.addedKeys.length as number, + updated: summary?.updatedKeys.length as number, + deleted: summary?.deletedKeys?.length as number + } + }; + + await createActivityLogMessage({ + level: 'info', + environment_id: this.environmentId as number, + activity_log_id: this.activityLogId as number, + content: `Batch update was a success and resulted in ${JSON.stringify(updatedResults, null, 2)}`, + timestamp: Date.now() + }); + + await updateSyncJobResult(this.syncJobId as number, updatedResults, model); + + return true; + } else { + const content = `There was an issue with the batch update. ${responseResults?.error}`; + + if (!this.dryRun) { + await createActivityLogMessage({ + level: 'error', + environment_id: this.environmentId as number, + activity_log_id: this.activityLogId as number, + content, + timestamp: Date.now() + }); + + await errorManager.report(content, { + environmentId: this.environmentId as number, + source: ErrorSourceEnum.CUSTOMER, + operation: LogActionEnum.SYNC, + metadata: { + connectionId: this.connectionId, + providerConfigKey: this.providerConfigKey, + syncId: this.syncId, + nanogConnectionId: this.nangoConnectionId, + syncJobId: this.syncJobId + } + }); + } + + throw new Error(responseResults?.error); + } + } + public override async getMetadata(): Promise { if (this.dryRun && this.stubbedMetadata) { return this.stubbedMetadata as T; diff --git a/packages/shared/lib/services/config.service.ts b/packages/shared/lib/services/config.service.ts index 759e85821d9..f9c65e74c08 100644 --- a/packages/shared/lib/services/config.service.ts +++ b/packages/shared/lib/services/config.service.ts @@ -9,6 +9,7 @@ import { NangoError } from '../utils/error.js'; import encryptionManager from '../utils/encryption.manager.js'; import syncOrchestrator from './sync/orchestrator.service.js'; import { deleteSyncFilesForConfig, deleteByConfigId as deleteSyncConfigByConfigId } from '../services/sync/config/config.service.js'; +import environmentService from '../services/environment.service.js'; class ConfigService { templates: { [key: string]: ProviderTemplate } | null; @@ -81,6 +82,23 @@ class ConfigService { return result[0].id; } + async getProviderConfigByUuid(providerConfigKey: string, environment_uuid: string): Promise { + if (!providerConfigKey) { + throw new NangoError('missing_provider_config'); + } + if (!environment_uuid) { + throw new NangoError('missing_environment_uuid'); + } + + const environment_id = await environmentService.getIdByUuid(environment_uuid); + + if (!environment_id) { + return null; + } + + return this.getProviderConfig(providerConfigKey, environment_id); + } + async getProviderConfig(providerConfigKey: string, environment_id: number): Promise { if (!providerConfigKey) { throw new NangoError('missing_provider_config'); diff --git a/packages/shared/lib/services/connection.service.integration.test.ts b/packages/shared/lib/services/connection.service.integration.test.ts new file mode 100644 index 00000000000..0c95b439b49 --- /dev/null +++ b/packages/shared/lib/services/connection.service.integration.test.ts @@ -0,0 +1,59 @@ +import { expect, describe, it, beforeAll } from 'vitest'; +import { multipleMigrations } from '../db/database.js'; +import connectionService from './connection.service.js'; +import type { Connection, Metadata } from '../models/Connection.js'; +import { createConfigSeeds } from '../db/seeders/config.seeder.js'; +import { createConnectionSeeds } from '../db/seeders/connection.seeder.js'; + +describe('Connection service integration tests', () => { + beforeAll(async () => { + await multipleMigrations(); + await createConfigSeeds(); + }); + + describe('Metadata simple operations', () => { + it('Should replace existing metadata, overwriting anything existing', async () => { + const connections = await createConnectionSeeds(); + + const initialMetadata = { + name: 'test', + host: 'test' + }; + + const newMetadata = { + additionalName: 'test23' + }; + + const [connectionId] = connections; + const connection = { id: connectionId } as Connection; + await connectionService.replaceMetadata(connection, initialMetadata); + await connectionService.replaceMetadata(connection, newMetadata); + + const dbConnection = await connectionService.getConnectionById(connectionId as number); + const updatedMetadata = dbConnection?.metadata as Metadata; + expect(updatedMetadata).toEqual(newMetadata); + }); + + it('Should update metadata and not overwrite', async () => { + const connections = await createConnectionSeeds(); + + const initialMetadata = { + name: 'test', + host: 'test' + }; + + const newMetadata = { + additionalName: 'test23' + }; + + const connectionId = connections[1]; + const dbConnection = (await connectionService.getConnectionById(connectionId as number)) as Connection; + await connectionService.replaceMetadata(dbConnection, initialMetadata); + await connectionService.updateMetadata(dbConnection, newMetadata); + + const updatedDbConnection = await connectionService.getConnectionById(connectionId as number); + const updatedMetadata = updatedDbConnection?.metadata as Metadata; + expect(updatedMetadata).toEqual({ ...initialMetadata, ...newMetadata }); + }); + }); +}); diff --git a/packages/shared/lib/services/connection.service.ts b/packages/shared/lib/services/connection.service.ts index 2aab67e7be4..b04956f4f73 100644 --- a/packages/shared/lib/services/connection.service.ts +++ b/packages/shared/lib/services/connection.service.ts @@ -25,7 +25,7 @@ import environmentService from '../services/environment.service.js'; import { getFreshOAuth2Credentials } from '../clients/oauth2.client.js'; import { NangoError } from '../utils/error.js'; -import type { Metadata, Connection, StoredConnection, BaseConnection, NangoConnection } from '../models/Connection.js'; +import type { Metadata, ConnectionConfig, Connection, StoredConnection, BaseConnection, NangoConnection } from '../models/Connection.js'; import type { ServiceResponse } from '../models/Generic.js'; import encryptionManager from '../utils/encryption.manager.js'; import metricsManager, { MetricTypes } from '../utils/metrics.manager.js'; @@ -39,7 +39,7 @@ import { } from '../models/Auth.js'; import { schema } from '../db/database.js'; import { interpolateStringFromObject, parseTokenExpirationDate, isTokenExpired, getRedisUrl } from '../utils/utils.js'; -import SyncClient from '../clients/sync.client.js'; +import { connectionCreated as connectionCreatedHook } from '../hooks/hooks.js'; import { Locking } from '../utils/lock/locking.js'; import { InMemoryKVStore } from '../utils/kvstore/InMemoryStore.js'; import { RedisKVStore } from '../utils/kvstore/RedisStore.js'; @@ -61,7 +61,7 @@ class ConnectionService { environment_id: number, accountId: number, metadata?: Metadata - ) { + ): Promise<{ id: number }[]> { const storedConnection = await this.checkIfConnectionExists(connectionId, providerConfigKey, environment_id); if (storedConnection) { @@ -208,8 +208,15 @@ class ConnectionService { ); if (importedConnection) { - const syncClient = await SyncClient.getInstance(); - syncClient?.initiate(importedConnection[0]?.id as number); + await connectionCreatedHook( + { + id: importedConnection[0]?.id as number, + connection_id, + provider_config_key, + environment_id: environmentId + }, + provider + ); } return importedConnection; @@ -232,8 +239,15 @@ class ConnectionService { const importedConnection = await this.upsertApiConnection(connection_id, provider_config_key, provider, credentials, {}, environmentId, accountId); if (importedConnection) { - const syncClient = await SyncClient.getInstance(); - syncClient?.initiate(importedConnection[0].id); + await connectionCreatedHook( + { + id: importedConnection[0].id, + connection_id, + provider_config_key, + environment_id: environmentId + }, + provider + ); } return importedConnection; @@ -376,11 +390,26 @@ class ConnectionService { return result[0].metadata; } + public async getConnectionConfig(connection: Connection): Promise { + const result = await db.knex.withSchema(db.schema()).from(`_nango_connections`).select('connection_config').where({ + connection_id: connection.connection_id, + provider_config_key: connection.provider_config_key, + environment_id: connection.environment_id, + deleted: false + }); + + if (!result || result.length == 0 || !result[0]) { + return {}; + } + + return result[0].connection_config; + } + public async getConnectionsByEnvironmentAndConfig(environment_id: number, providerConfigKey: string): Promise { const result = await db.knex .withSchema(db.schema()) .from(`_nango_connections`) - .select('id', 'connection_id', 'provider_config_key', 'environment_id') + .select('id', 'connection_id', 'provider_config_key', 'environment_id', 'connection_config') .where({ environment_id, provider_config_key: providerConfigKey, deleted: false }); if (!result || result.length == 0 || !result[0]) { @@ -390,7 +419,7 @@ class ConnectionService { return result; } - public async updateMetadata(connection: Connection, metadata: Metadata) { + public async replaceMetadata(connection: Connection, metadata: Metadata) { await db.knex .withSchema(db.schema()) .from(`_nango_connections`) @@ -398,6 +427,44 @@ class ConnectionService { .update({ metadata }); } + public async replaceConnectionConfig(connection: Connection, config: ConnectionConfig) { + await db.knex + .withSchema(db.schema()) + .from(`_nango_connections`) + .where({ id: connection.id as number, deleted: false }) + .update({ connection_config: config }); + } + + public async updateMetadata(connection: Connection, metadata: Metadata): Promise { + const existingMetadata = await this.getMetadata(connection); + const newMetadata = { ...existingMetadata, ...metadata }; + await this.replaceMetadata(connection, newMetadata); + + return newMetadata; + } + + public async updateConnectionConfig(connection: Connection, config: ConnectionConfig): Promise { + const existingConfig = await this.getConnectionConfig(connection); + const newConfig = { ...existingConfig, ...config }; + await this.replaceConnectionConfig(connection, newConfig); + + return newConfig; + } + + public async findConnectionByConnectionConfigValue(key: string, value: string): Promise { + const result = await db.knex + .withSchema(db.schema()) + .from(`_nango_connections`) + .select('*') + .whereRaw(`connection_config->>? = ? AND deleted = false`, [key, value]); + + if (!result || result.length == 0 || !result[0]) { + return null; + } + + return encryptionManager.decryptConnection(result[0]); + } + public async listConnections( environment_id: number, connectionId?: string diff --git a/packages/shared/lib/services/environment.service.ts b/packages/shared/lib/services/environment.service.ts index b1e273f7e68..01a87bd0fc4 100644 --- a/packages/shared/lib/services/environment.service.ts +++ b/packages/shared/lib/services/environment.service.ts @@ -133,6 +133,20 @@ class EnvironmentService { return uuid; } + async getAccountUUIDFromEnvironmentUUID(environment_uuid: string): Promise { + const result = await db.knex.withSchema(db.schema()).select('account_id').from(TABLE).where({ uuid: environment_uuid }); + + if (result == null || result.length == 0 || result[0] == null) { + return null; + } + + const accountId = result[0].account_id; + + const uuid = await accountService.getUUIDFromAccountId(accountId); + + return uuid; + } + async getAccountIdAndEnvironmentIdByPublicKey(publicKey: string): Promise<{ accountId: number; environmentId: number } | null> { if (!isCloud()) { const environmentVariables = Object.keys(process.env).filter((key) => key.startsWith('NANGO_PUBLIC_KEY_')) || []; @@ -201,6 +215,16 @@ class EnvironmentService { return { account: account[0], environment: encryptionManager.decryptEnvironment(environmentResult[0]) }; } + async getIdByUuid(uuid: string): Promise { + const result = await db.knex.withSchema(db.schema()).select('id').from(TABLE).where({ uuid }); + + if (result == null || result.length == 0 || result[0] == null) { + return null; + } + + return result[0].id; + } + async getById(id: number): Promise { try { const result = (await schema().select('*').from(TABLE).where({ id })) as unknown as Environment[]; diff --git a/packages/shared/lib/services/nango-config.service.ts b/packages/shared/lib/services/nango-config.service.ts index 86ccb11e0af..14511bc0f4e 100644 --- a/packages/shared/lib/services/nango-config.service.ts +++ b/packages/shared/lib/services/nango-config.service.ts @@ -269,6 +269,7 @@ export function convertV2ConfigObject(config: NangoConfigV2, showMessages = fals for (const providerConfigKey in config.integrations) { const builtSyncs: NangoSyncConfig[] = []; const builtActions: NangoSyncConfig[] = []; + const integration: NangoV2Integration = config.integrations[providerConfigKey] as NangoV2Integration; let provider; @@ -283,6 +284,7 @@ export function convertV2ConfigObject(config: NangoConfigV2, showMessages = fals const syncs = integration['syncs'] as NangoV2Integration; const actions = integration['actions'] as NangoV2Integration; + for (const syncName in syncs) { const sync: NangoIntegrationDataV2 = syncs[syncName] as NangoIntegrationDataV2; const models: NangoSyncModel[] = []; @@ -358,6 +360,16 @@ export function convertV2ConfigObject(config: NangoConfigV2, showMessages = fals return { success: false, error, response: null }; } + let webhookSubscriptions: string[] = []; + + if (sync['webhook-subscriptions']) { + if (Array.isArray(sync['webhook-subscriptions'])) { + webhookSubscriptions = sync['webhook-subscriptions'] as string[]; + } else { + webhookSubscriptions = [sync['webhook-subscriptions'] as string]; + } + } + const syncObject: NangoSyncConfig = { name: syncName, type: SyncConfigType.SYNC, @@ -372,7 +384,8 @@ export function convertV2ConfigObject(config: NangoConfigV2, showMessages = fals returns: Array.isArray(sync.output) ? (sync?.output as string[]) : ([sync.output] as string[]), description: sync?.description || sync?.metadata?.description || '', scopes: Array.isArray(scopes) ? scopes : String(scopes)?.split(','), - endpoints + endpoints, + webhookSubscriptions }; builtSyncs.push(syncObject); diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts index 65161e8c4a8..416da5d9027 100644 --- a/packages/shared/lib/services/sync/config/config.service.ts +++ b/packages/shared/lib/services/sync/config/config.service.ts @@ -44,8 +44,10 @@ export async function getSyncConfig(nangoConnection: NangoConnection, syncName?: const key = nangoConnection.provider_config_key; const providerConfig = nangoConfig.integrations[key] ?? {}; + const configSyncName = syncConfig.sync_name; + const fileLocation = syncConfig.file_location; - providerConfig[syncConfig.sync_name] = { + providerConfig[configSyncName] = { sync_config_id: syncConfig.id as number, runs: syncConfig.runs, type: syncConfig.type, @@ -54,7 +56,7 @@ export async function getSyncConfig(nangoConnection: NangoConnection, syncName?: track_deletes: syncConfig.track_deletes, auto_start: syncConfig.auto_start, attributes: syncConfig.attributes || {}, - fileLocation: syncConfig.file_location, + fileLocation, version: syncConfig.version as string, pre_built: syncConfig.pre_built as boolean, is_public: syncConfig.is_public as boolean, @@ -206,11 +208,15 @@ export async function getSyncConfigsByParams(environment_id: number, providerCon throw new Error('Provider config not found'); } + return getSyncConfigsByConfigId(environment_id, config.id as number, isAction); +} + +export async function getSyncConfigsByConfigId(environment_id: number, nango_config_id: number, isAction = false): Promise { const result = await schema() .from(TABLE) .where({ environment_id, - nango_config_id: config.id as number, + nango_config_id, active: true, type: isAction ? SyncConfigType.ACTION : SyncConfigType.SYNC, deleted: false diff --git a/packages/shared/lib/services/sync/config/deploy.service.ts b/packages/shared/lib/services/sync/config/deploy.service.ts index aaa1755594c..dfbdc7371a0 100644 --- a/packages/shared/lib/services/sync/config/deploy.service.ts +++ b/packages/shared/lib/services/sync/config/deploy.service.ts @@ -35,6 +35,8 @@ const ENDPOINT_TABLE = dbNamespace + 'sync_endpoints'; const nameOfType = 'sync/action'; +type FlowWithVersion = Omit; + export async function deploy( environment_id: number, flows: IncomingFlowConfig[], @@ -65,7 +67,7 @@ export async function deploy( operation_name: LogActionEnum.SYNC_DEPLOY }; - let flowsWithVersions: Omit[] = flows.map((flow) => { + let flowsWithVersions: FlowWithVersion[] = flows.map((flow) => { const { fileBody: _fileBody, model_schema, ...rest } = flow; const modelSchema = JSON.parse(model_schema); return { ...rest, model_schema: modelSchema }; @@ -81,149 +83,24 @@ export async function deploy( const flowReturnData: SyncDeploymentResult[] = []; for (const flow of flows) { - const { - syncName, - providerConfigKey, - fileBody, - models, - runs, - version: optionalVersion, - model_schema, - type = SyncConfigType.SYNC, - track_deletes, - auto_start, - attributes = {}, - metadata = {} - } = flow; - if (type === SyncConfigType.SYNC && !runs) { - const error = new NangoError('missing_required_fields_on_deploy'); - - return { success: false, error, response: null }; - } - - if (!syncName || !providerConfigKey || !fileBody) { - const error = new NangoError('missing_required_fields_on_deploy'); - - return { success: false, error, response: null }; - } - - const config = await configService.getProviderConfig(providerConfigKey, environment_id); - - if (!config) { - const error = new NangoError('unknown_provider_config', { providerConfigKey }); - - return { success: false, error, response: null }; - } - - const previousSyncAndActionConfig = await getSyncAndActionConfigByParams(environment_id, syncName, providerConfigKey); - let bumpedVersion = ''; - - if (previousSyncAndActionConfig) { - bumpedVersion = increment(previousSyncAndActionConfig.version as string | number).toString(); - - if (debug) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `A previous sync config was found for ${syncName} with version ${previousSyncAndActionConfig.version}` - }); - } - - const syncs = await getSyncsByProviderConfigAndSyncName(environment_id, providerConfigKey, syncName); - for (const sync of syncs) { - if (!runs) { - continue; - } - const { success, error } = await updateSyncScheduleFrequency(sync.id as string, runs, syncName, activityLogId as number, environment_id); - - if (!success) { - return { success, error, response: null }; - } - } - } - - const version = optionalVersion || bumpedVersion || '1'; - - const jsFile = typeof fileBody === 'string' ? fileBody : fileBody?.js; - const file_location = (await remoteFileService.upload( - jsFile as string, - `${env}/account/${accountId}/environment/${environment_id}/config/${config.id}/${syncName}-v${version}.js`, - environment_id - )) as string; - - if (typeof fileBody === 'object' && fileBody?.ts) { - await remoteFileService.upload( - fileBody.ts, - `${env}/account/${accountId}/environment/${environment_id}/config/${config.id}/${syncName}.ts`, - environment_id - ); - } - - flowsWithVersions = flowsWithVersions.map((flowWithVersions) => { - if (flowWithVersions['syncName'] === syncName) { - return { ...flowWithVersions, version }; - } - return flowWithVersions; - }); - - if (!file_location) { - await updateSuccessActivityLog(activityLogId as number, false); - - await createActivityLogMessageAndEnd({ - level: 'error', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `There was an error uploading the sync file ${syncName}-v${version}.js` - }); - - // this is a platform error so throw this - throw new NangoError('file_upload_error'); - } - - const oldConfigs = await getSyncAndActionConfigsBySyncNameAndConfigId(environment_id, config.id as number, syncName); - - if (oldConfigs.length > 0) { - const ids = oldConfigs.map((oldConfig: SyncConfig) => oldConfig.id as number); - idsToMarkAsInvactive.push(...ids); + const { success, error, response } = await compileDeployInfo( + flow, + flowsWithVersions, + idsToMarkAsInvactive, + insertData, + flowReturnData, + env, + environment_id, + accountId, + activityLogId as number, + debug + ); - if (debug) { - await createActivityLogMessage({ - level: 'debug', - environment_id, - activity_log_id: activityLogId as number, - timestamp: Date.now(), - content: `Marking ${ids.length} old sync configs as inactive for ${syncName} with version ${version} as the active sync config` - }); - } + if (!success || !response) { + return { success, error, response: null }; } - insertData.push({ - environment_id, - nango_config_id: config?.id as number, - sync_name: syncName, - type, - models, - version, - track_deletes: track_deletes || false, - auto_start: auto_start === false ? false : true, - attributes, - metadata, - file_location, - runs, - active: true, - model_schema: model_schema as unknown as SyncModelSchema[], - input: flow.input || '', - sync_type: flow.sync_type - }); - - flowReturnData.push({ - ...flow, - name: syncName, - version - }); + flowsWithVersions = response as FlowWithVersion[]; } if (insertData.length === 0) { @@ -623,3 +500,166 @@ export async function deployPreBuilt( throw new NangoError('error_creating_sync_config'); } } + +async function compileDeployInfo( + flow: IncomingFlowConfig, + flowsWithVersions: FlowWithVersion[], + idsToMarkAsInvactive: number[], + insertData: SyncConfig[], + flowReturnData: SyncConfigResult['result'], + env: string, + environment_id: number, + accountId: number, + activityLogId: number, + debug: boolean +): Promise> { + const { + syncName, + providerConfigKey, + fileBody, + models, + runs, + version: optionalVersion, + model_schema, + type = SyncConfigType.SYNC, + track_deletes, + auto_start, + attributes = {}, + metadata = {} + } = flow; + if (type === SyncConfigType.SYNC && !runs) { + const error = new NangoError('missing_required_fields_on_deploy'); + + return { success: false, error, response: null }; + } + + if (!syncName || !providerConfigKey || !fileBody) { + const error = new NangoError('missing_required_fields_on_deploy'); + + return { success: false, error, response: null }; + } + + const config = await configService.getProviderConfig(providerConfigKey, environment_id); + + if (!config) { + const error = new NangoError('unknown_provider_config', { providerConfigKey }); + + return { success: false, error, response: null }; + } + + const previousSyncAndActionConfig = await getSyncAndActionConfigByParams(environment_id, syncName, providerConfigKey); + let bumpedVersion = ''; + + if (previousSyncAndActionConfig) { + bumpedVersion = increment(previousSyncAndActionConfig.version as string | number).toString(); + + if (debug) { + await createActivityLogMessage({ + level: 'debug', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `A previous sync config was found for ${syncName} with version ${previousSyncAndActionConfig.version}` + }); + } + + const syncs = await getSyncsByProviderConfigAndSyncName(environment_id, providerConfigKey, syncName); + for (const sync of syncs) { + if (!runs) { + continue; + } + const { success, error } = await updateSyncScheduleFrequency(sync.id as string, runs, syncName, activityLogId as number, environment_id); + + if (!success) { + return { success, error, response: null }; + } + } + } + + const version = optionalVersion || bumpedVersion || '1'; + + const jsFile = typeof fileBody === 'string' ? fileBody : fileBody?.js; + const file_location = (await remoteFileService.upload( + jsFile as string, + `${env}/account/${accountId}/environment/${environment_id}/config/${config.id}/${syncName}-v${version}.js`, + environment_id + )) as string; + + if (typeof fileBody === 'object' && fileBody?.ts) { + await remoteFileService.upload( + fileBody.ts, + `${env}/account/${accountId}/environment/${environment_id}/config/${config.id}/${syncName}.ts`, + environment_id + ); + } + + flowsWithVersions = flowsWithVersions.map((flowWithVersions) => { + if (flowWithVersions['syncName'] === syncName) { + return { + ...flowWithVersions, + version + } as unknown as FlowWithVersion; + } + return flowWithVersions; + }); + + if (!file_location) { + await updateSuccessActivityLog(activityLogId as number, false); + + await createActivityLogMessageAndEnd({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `There was an error uploading the sync file ${syncName}-v${version}.js` + }); + + // this is a platform error so throw this + throw new NangoError('file_upload_error'); + } + + const oldConfigs = await getSyncAndActionConfigsBySyncNameAndConfigId(environment_id, config.id as number, syncName); + + if (oldConfigs.length > 0) { + const ids = oldConfigs.map((oldConfig: SyncConfig) => oldConfig.id as number); + idsToMarkAsInvactive.push(...ids); + + if (debug) { + await createActivityLogMessage({ + level: 'debug', + environment_id, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `Marking ${ids.length} old sync configs as inactive for ${syncName} with version ${version} as the active sync config` + }); + } + } + + insertData.push({ + environment_id, + nango_config_id: config?.id as number, + sync_name: syncName, + type, + models, + version, + track_deletes: track_deletes || false, + auto_start: auto_start === false ? false : true, + attributes, + metadata, + file_location, + runs, + active: true, + model_schema: model_schema as unknown as SyncModelSchema[], + input: flow.input || '', + sync_type: flow.sync_type, + webhook_subscriptions: flow.webhookSubscriptions || [] + }); + + flowReturnData.push({ + ...flow, + name: syncName, + version + }); + + return { success: true, error: null, response: flowsWithVersions }; +} diff --git a/packages/shared/lib/services/sync/data/data.service.ts b/packages/shared/lib/services/sync/data/data.service.ts index ceeb5a0d37c..431644fd593 100644 --- a/packages/shared/lib/services/sync/data/data.service.ts +++ b/packages/shared/lib/services/sync/data/data.service.ts @@ -1,5 +1,5 @@ import { schema } from '../../../db/database.js'; -import { verifyUniqueKeysAreUnique } from './records.service.js'; +import { getRecordsByExternalIds, verifyUniqueKeysAreUnique } from './records.service.js'; import { createActivityLogMessage } from '../../activity/activity.service.js'; import { markRecordsForDeletion, syncCreatedAtForAddedRecords, syncUpdateAtForChangedRecords } from './delete.service.js'; import type { UpsertResponse } from '../../../models/Data.js'; @@ -97,6 +97,92 @@ export async function upsert( } } +export async function updateRecord( + records: DataRecord[], + dbTable: string, + uniqueKey: string, + nangoConnectionId: number, + model: string, + activityLogId: number, + environment_id: number +): Promise { + const responseWithoutDuplicates = await removeDuplicateKey(records, uniqueKey, activityLogId, environment_id, model); + + if (!responseWithoutDuplicates || responseWithoutDuplicates.length === 0) { + return { + success: false, + error: `There are no records to upsert because there were no records that were not duplicates to insert, but there were ${records.length} records received for the "${model}" model.` + }; + } + + const updatedKeys = await getUpdatedKeys(responseWithoutDuplicates, dbTable, uniqueKey, nangoConnectionId, model); + + try { + const recordsToUpdate = []; + const rawOldRecords = await getRecordsByExternalIds(updatedKeys, nangoConnectionId, model); + + for (const rawOldRecord of rawOldRecords) { + if (!rawOldRecord) { + continue; + } + + const { record: oldRecord } = rawOldRecord; + + const record = records.find((record) => record.external_id.toString() === (oldRecord as DataRecord)?.id?.toString()); + + const newRecord = { + ...rawOldRecord, + json: { + ...oldRecord, + ...record?.json + }, + updated_at: new Date() + }; + + delete newRecord.record; + + recordsToUpdate.push(newRecord); + } + + const encryptedRecords = encryptionManager.encryptDataRecords(recordsToUpdate); + + const results = await schema() + .from(dbTable) + .insert(encryptedRecords, ['id', 'external_id']) + .onConflict(['nango_connection_id', 'external_id', 'model']) + .merge() + .returning(['id', 'external_id']); + + const affectedInternalIds = results.map((tuple) => tuple.id) as string[]; + const affectedExternalIds = results.map((tuple) => tuple.external_id) as string[]; + + return { + success: true, + summary: { + addedKeys: [], + updatedKeys, + deletedKeys: [], + affectedInternalIds, + affectedExternalIds + } + }; + } catch (error: any) { + let errorMessage = `Failed to update records to table ${dbTable}.\n`; + errorMessage += `Model: ${model}, Unique Key: ${uniqueKey}, Nango Connection ID: ${nangoConnectionId}.\n`; + errorMessage += `Attempted to update: ${responseWithoutDuplicates.length} records\n`; + + if ('code' in error) errorMessage += `Error code: ${error.code}.\n`; + if ('detail' in error) errorMessage += `Detail: ${error.detail}.\n`; + + errorMessage += `Error Message: ${error.message}`; + + return { + success: false, + error: errorMessage + }; + } +} + export async function removeDuplicateKey( response: DataRecord[], uniqueKey: string, diff --git a/packages/shared/lib/services/sync/data/records.service.ts b/packages/shared/lib/services/sync/data/records.service.ts index 474205b4dfa..aa7a470cb65 100644 --- a/packages/shared/lib/services/sync/data/records.service.ts +++ b/packages/shared/lib/services/sync/data/records.service.ts @@ -525,3 +525,45 @@ export async function deleteRecordsBySyncId(sync_id: string): Promise { await schema().from('_nango_sync_data_records').where({ sync_id }).del(); await schema().from('_nango_sync_data_records_deletes').where({ sync_id }).del(); } + +export async function getSingleRecord(external_id: string, nango_connection_id: number, model: string): Promise { + const encryptedRecord = await schema().from('_nango_sync_data_records').where({ + nango_connection_id, + model, + external_id + }); + + if (!encryptedRecord) { + return null; + } + + const result = encryptionManager.decryptDataRecords(encryptedRecord, 'json'); + + if (!result || result.length === 0) { + return null; + } + + return result[0] as unknown as SyncDataRecord; +} + +export async function getRecordsByExternalIds(external_ids: string[], nango_connection_id: number, model: string): Promise { + const encryptedRecords = await schema() + .from('_nango_sync_data_records') + .where({ + nango_connection_id, + model + }) + .whereIn('external_id', external_ids); + + if (!encryptedRecords) { + return []; + } + + const result = encryptionManager.decryptDataRecords(encryptedRecords, 'json'); + + if (!result || result.length === 0) { + return []; + } + + return result as unknown as SyncDataRecord[]; +} diff --git a/packages/shared/lib/services/sync/notification/webhook.service.ts b/packages/shared/lib/services/sync/notification/webhook.service.ts index 33861b33be8..c81410c29bb 100644 --- a/packages/shared/lib/services/sync/notification/webhook.service.ts +++ b/packages/shared/lib/services/sync/notification/webhook.service.ts @@ -2,9 +2,10 @@ import axios, { AxiosError } from 'axios'; import { backOff } from 'exponential-backoff'; import { SyncType } from '../../../models/Sync.js'; import type { NangoConnection } from '../../../models/Connection'; +import { LogActionEnum, LogLevel } from '../../../models/Activity.js'; import type { SyncResult, NangoSyncWebhookBody } from '../../../models/Sync'; import environmentService from '../../environment.service.js'; -import { createActivityLogMessage } from '../../activity/activity.service.js'; +import { createActivityLog, createActivityLogMessage } from '../../activity/activity.service.js'; const RETRY_ATTEMPTS = 10; @@ -63,6 +64,7 @@ class WebhookService { } const body: NangoSyncWebhookBody = { + from: 'nango', connectionId: nangoConnection.connection_id, providerConfigKey: nangoConnection.provider_config_key, syncName, @@ -125,6 +127,77 @@ class WebhookService { }); } } + + async forward(environment_id: number, providerConfigKey: string, provider: string, payload: unknown) { + const webhookInfo = await environmentService.getWebhookInfo(environment_id); + + if (!webhookInfo || !webhookInfo.webhook_url) { + return; + } + + const { webhook_url: webhookUrl } = webhookInfo; + + const log = { + level: 'info' as LogLevel, + success: null, + action: LogActionEnum.SYNC, + start: Date.now(), + end: Date.now(), + timestamp: Date.now(), + connection_id: '', + provider_config_key: providerConfigKey, + provider: provider, + environment_id: environment_id + }; + + const activityLogId = await createActivityLog(log); + + const body = { + from: provider, + payload + }; + + try { + const response = await backOff( + () => { + return axios.post(webhookUrl, body); + }, + { numOfAttempts: RETRY_ATTEMPTS, retry: this.retry.bind(this, activityLogId as number, environment_id) } + ); + + if (response.status >= 200 && response.status < 300) { + await createActivityLogMessage({ + level: 'info', + environment_id, + activity_log_id: activityLogId as number, + content: `Webhook sent successfully and received with a ${ + response.status + } response code to ${webhookUrl} with the following data: ${JSON.stringify(body, null, 2)}`, + timestamp: Date.now() + }); + } else { + await createActivityLogMessage({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + content: `Webhook sent successfully to ${webhookUrl} with the following data: ${JSON.stringify(body, null, 2)} but received a ${ + response.status + } response code. Please send a 200 on successful receipt.`, + timestamp: Date.now() + }); + } + } catch (e) { + const errorMessage = JSON.stringify(e, ['message', 'name', 'stack'], 2); + + await createActivityLogMessage({ + level: 'error', + environment_id, + activity_log_id: activityLogId as number, + content: `Webhook failed to send to ${webhookUrl}. The error was: ${errorMessage}`, + timestamp: Date.now() + }); + } + } } export default new WebhookService(); diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 421161a6282..dc2c547c100 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -28,6 +28,8 @@ interface SyncRunConfig { integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction?: boolean; + isInvokedImmediately?: boolean; + isWebhook?: boolean; nangoConnection: NangoConnection; syncName: string; syncType: SyncType; @@ -51,6 +53,7 @@ export default class SyncRun { integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction: boolean; + isInvokedImmediately: boolean; nangoConnection: NangoConnection; syncName: string; syncType: SyncType; @@ -67,14 +70,17 @@ export default class SyncRun { stubbedMetadata?: Metadata | undefined = undefined; temporalContext?: Context; + isWebhook: boolean; constructor(config: SyncRunConfig) { this.integrationService = config.integrationService; this.writeToDb = config.writeToDb; this.isAction = config.isAction || false; + this.isWebhook = config.isWebhook || false; this.nangoConnection = config.nangoConnection; this.syncName = config.syncName; this.syncType = config.syncType; + this.isInvokedImmediately = Boolean(config.isAction || config.isWebhook); if (config.syncId) { this.syncId = config.syncId; @@ -149,7 +155,7 @@ export default class SyncRun { console.error(message); } - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -159,7 +165,7 @@ export default class SyncRun { if (!integrations[this.nangoConnection.provider_config_key] && !this.writeToDb) { const message = `The connection you provided which applies to integration "${this.nangoConnection.provider_config_key}" does not match any integration in the ${nangoConfigFile}`; - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -174,7 +180,7 @@ export default class SyncRun { if (!environment && !bypassEnvironment) { const message = `No environment was found for ${this.nangoConnection.environment_id}. The sync cannot continue without a valid environment`; await this.reportFailureForResults(message); - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -223,7 +229,7 @@ export default class SyncRun { const message = `Integration was attempted to run for ${this.syncName} but no integration file was found at ${integrationFilePath}.`; await this.reportFailureForResults(message); - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -231,7 +237,7 @@ export default class SyncRun { let lastSyncDate: Date | null | undefined = null; - if (!this.isAction) { + if (!this.isInvokedImmediately) { if (!this.writeToDb) { lastSyncDate = optionalLastSyncDate; } else { @@ -310,7 +316,8 @@ export default class SyncRun { syncData, this.nangoConnection.environment_id, this.writeToDb, - this.isAction, + this.isInvokedImmediately, + this.isWebhook, this.loadLocation, this.input, this.temporalContext @@ -459,7 +466,7 @@ export default class SyncRun { const message = `There was a problem upserting the data for ${this.syncName} and the model ${model} with the error message: ${upsertResult?.error}`; await this.reportFailureForResults(message); - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message), response: result }; } @@ -479,7 +486,7 @@ export default class SyncRun { } sync did not complete successfully and has the following error: ${errorMessage}` ); - const errorType = this.isAction ? 'action_script_failure' : 'sync_script_failure'; + const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, errorMessage), response: result }; } @@ -491,12 +498,12 @@ export default class SyncRun { async finishSync(models: string[], syncStartDate: Date, version: string, totalRunTime: number, trackDeletes?: boolean): Promise { let i = 0; for (const model of models) { - if (trackDeletes) { + if (!this.isWebhook && trackDeletes) { await clearOldRecords(this.nangoConnection?.id as number, model); } const deletedKeys = trackDeletes ? await getDeletedKeys('_nango_sync_data_records', 'external_id', this.nangoConnection.id as number, model) : []; - if (trackDeletes) { + if (!this.isWebhook && trackDeletes) { await syncUpdateAtForDeletedRecords(this.nangoConnection.id as number, model, 'external_id', deletedKeys); } @@ -536,19 +543,21 @@ export default class SyncRun { // any changes while the sync is running // but if the sync date was set by the user in the integration script, // then don't override it - const override = false; - await setLastSyncDate(this.syncId as string, syncStartDate, override); - await slackNotificationService.removeFailingConnection( - this.nangoConnection, - this.syncName, - this.syncType, - this.activityLogId as number, - this.nangoConnection.environment_id, - this.provider as string - ); + if (!this.isWebhook) { + const override = false; + await setLastSyncDate(this.syncId as string, syncStartDate, override); + await slackNotificationService.removeFailingConnection( + this.nangoConnection, + this.syncName, + this.syncType, + this.activityLogId as number, + this.nangoConnection.environment_id, + this.provider as string + ); + } } - if (trackDeletes) { + if (!this.isWebhook && trackDeletes) { await takeSnapshot(this.nangoConnection?.id as number, model); } @@ -662,14 +671,16 @@ export default class SyncRun { return; } - await slackNotificationService.reportFailure( - this.nangoConnection, - this.syncName, - this.syncType, - this.activityLogId as number, - this.nangoConnection.environment_id, - this.provider as string - ); + if (!this.isWebhook) { + await slackNotificationService.reportFailure( + this.nangoConnection, + this.syncName, + this.syncType, + this.activityLogId as number, + this.nangoConnection.environment_id, + this.provider as string + ); + } if (!this.activityLogId || !this.syncJobId) { console.error(content); @@ -717,4 +728,14 @@ export default class SyncRun { `syncId:${this.syncId}` ); } + + private determineErrorType(): string { + if (this.isAction) { + return 'action_script_failure'; + } else if (this.isWebhook) { + return 'webhook_script_failure'; + } else { + return 'sync_script_failure'; + } + } } diff --git a/packages/shared/lib/utils/error.ts b/packages/shared/lib/utils/error.ts index 213cf176438..0a74db2611d 100644 --- a/packages/shared/lib/utils/error.ts +++ b/packages/shared/lib/utils/error.ts @@ -444,6 +444,10 @@ export class NangoError extends Error { this.message = `The action script failed with an error: ${this.payload}`; break; + case 'webhook_script_failure': + this.message = `The webhook script failed with an error: ${this.payload}`; + break; + case 'pass_through_error': this.status = 400; this.message = `${this.payload}`; diff --git a/packages/shared/lib/utils/metrics.manager.ts b/packages/shared/lib/utils/metrics.manager.ts index 3431e0e40ee..f94cf26ab3e 100644 --- a/packages/shared/lib/utils/metrics.manager.ts +++ b/packages/shared/lib/utils/metrics.manager.ts @@ -24,7 +24,9 @@ export enum MetricTypes { SYNC_GET_RECORDS_INCLUDE_METADATA_USED = 'sync_get_records_include_metadata_used', SYNC_GET_RECORDS_DEPRECATED_METHOD_USED = 'sync_get_records_deprecated_method_used', SYNC_GET_RECORDS_QUERY_TIMEOUT = 'sync_get_records_query_timeout', - FLOW_JOB_TIMEOUT_FAILURE = 'flow_job_failure' + FLOW_JOB_TIMEOUT_FAILURE = 'flow_job_failure', + POST_CONNECTION_SCRIPT_FAILURE = 'post_connection_script_failure', + INCOMING_WEBHOOK_RECEIVED = 'incoming_webhook_received' } class MetricsManager { diff --git a/packages/shared/lib/utils/utils.ts b/packages/shared/lib/utils/utils.ts index d878c0739e7..398dd5963fa 100644 --- a/packages/shared/lib/utils/utils.ts +++ b/packages/shared/lib/utils/utils.ts @@ -176,6 +176,11 @@ export function getGlobalAppCallbackUrl() { return baseUrl + '/app-auth/connect'; } +export function getGlobalWebhookReceiveUrl() { + const baseUrl = process.env['NANGO_SERVER_URL'] || getLocalOAuthCallbackUrlBaseUrl(); + return baseUrl + '/webhook'; +} + export async function getOauthCallbackUrl(environmentId?: number) { const globalCallbackUrl = getGlobalOAuthCallbackUrl(); diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index 8a2dffce504..fc4c65e0106 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -570,6 +570,10 @@ hubspot: auth_mode: OAUTH2 authorization_url: https://app.hubspot.com/oauth/authorize token_url: https://api.hubapi.com/oauth/v1/token + connection_configuration: + - portalId + post_connection_script: hubspot-post-connection + webhook_routing_script: hubspot-webhook-routing proxy: base_url: https://api.hubapi.com decompress: true @@ -605,6 +609,9 @@ jira: authorization_params: audience: api.atlassian.com prompt: consent + connection_configuration: + - cloudID + - accountId proxy: base_url: https://api.atlassian.com paginate: @@ -613,6 +620,8 @@ jira: limit_name_in_request: limit response_path: results link_path_in_response_body: _links.next + post_connection_script: jira-post-connection + webhook_routing_script: jira-webhook-routing keap: auth_mode: OAUTH2 authorization_url: https://accounts.infusionsoft.com/app/oauth/authorize diff --git a/packages/webapp/src/pages/Activity.tsx b/packages/webapp/src/pages/Activity.tsx index 1d27bbd4784..6c0e2090a81 100644 --- a/packages/webapp/src/pages/Activity.tsx +++ b/packages/webapp/src/pages/Activity.tsx @@ -332,6 +332,21 @@ export default function Activity() { )} )} + {activity?.action === 'webhook' && ( +
+
+ +

webhook

+
+ {activity.endpoint && ( + +
+ {activity.endpoint} +
+ + )} +
+ )} {activity?.action === 'sync' && (
diff --git a/packages/webapp/src/pages/IntegrationCreate.tsx b/packages/webapp/src/pages/IntegrationCreate.tsx index 247dda69cbb..0e748151a4b 100644 --- a/packages/webapp/src/pages/IntegrationCreate.tsx +++ b/packages/webapp/src/pages/IntegrationCreate.tsx @@ -48,6 +48,8 @@ export default function IntegrationCreate() { const { providerConfigKey } = useParams(); const [templateLogo, setTemplateLogo] = useState(''); const [callbackUrl, setCallbackUrl] = useState(''); + const [webhookReceiveUrl, setWebhookReceiveUrl] = useState(''); + const [hasWebhook, setHasWebhook] = useState(false); const getIntegrationDetailsAPI = useGetIntegrationDetailsAPI(); const getProvidersAPI = useGetProvidersAPI(); const getProjectInfoAPI = useGetProjectInfoAPI(); @@ -69,6 +71,7 @@ export default function IntegrationCreate() { if (currentIntegration['auth_mode']) { setAuthMode(currentIntegration['auth_mode']); } + setHasWebhook(currentIntegration['has_webhook']); } } else { let res = await getProvidersAPI(); @@ -89,6 +92,7 @@ export default function IntegrationCreate() { if (res?.status === 200) { const account = (await res.json())['account']; setCallbackUrl(account.callback_url || defaultCallback()); + setWebhookReceiveUrl(account.webhook_receive_url); } }; @@ -516,6 +520,31 @@ export default function IntegrationCreate() {
+ {providerConfigKey && hasWebhook && ( +
+
+
+ + +
+

{`Register this webhook URL on the developer portal of the Integration Provider to receive incoming webhooks.`}

+
+ + } + > + + +
+ + {`${webhookReceiveUrl}/${providerConfigKey}`} + +
+
+ )} )} diff --git a/packages/webapp/src/types.ts b/packages/webapp/src/types.ts index 6c089a91153..02381ad3fc0 100644 --- a/packages/webapp/src/types.ts +++ b/packages/webapp/src/types.ts @@ -1,7 +1,7 @@ export interface ActivityResponse { id: number; level: 'info' | 'debug' | 'error' | 'warn'; - action: 'account' | 'oauth' | 'auth' | 'proxy' | 'token' | 'sync' | 'sync deploy' | 'pause sync' | 'restart sync' | 'trigger sync' | 'action'; + action: 'account' | 'oauth' | 'auth' | 'proxy' | 'token' | 'sync' | 'sync deploy' | 'pause sync' | 'restart sync' | 'trigger sync' | 'action' | 'webhook'; success: boolean; timestamp: number; start: number; diff --git a/packages/worker/lib/activities.ts b/packages/worker/lib/activities.ts index c7cce9852b7..d7b91cd831d 100644 --- a/packages/worker/lib/activities.ts +++ b/packages/worker/lib/activities.ts @@ -23,10 +23,11 @@ import { MetricTypes, isInitialSyncStillRunning, initialSyncExists, + getSyncByIdAndName, logger } from '@nangohq/shared'; import integrationService from './integration.service.js'; -import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/Worker'; +import type { WebhookArgs, ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/Worker'; export async function routeSync(args: InitialSyncArgs): Promise { const { syncId, syncJobId, syncName, activityLogId, nangoConnection, debug } = args; @@ -81,6 +82,48 @@ export async function runAction(args: ActionArgs): Promise { return actionResults; } +export async function runWebhook(args: WebhookArgs): Promise { + const { input, nangoConnection, activityLogId, parentSyncName } = args; + + const syncConfig: ProviderConfig = (await configService.getProviderConfig( + nangoConnection?.provider_config_key as string, + nangoConnection?.environment_id as number + )) as ProviderConfig; + + const sync = await getSyncByIdAndName(nangoConnection.id as number, parentSyncName); + + const context: Context = Context.current(); + + const syncJobId = await createSyncJob( + sync?.id as string, + SyncType.WEBHOOK, + SyncStatus.RUNNING, + context.info.workflowExecution.workflowId, + nangoConnection, + context.info.workflowExecution.runId + ); + + const syncRun = new syncRunService({ + integrationService, + writeToDb: true, + nangoConnection, + syncJobId: syncJobId?.id as number, + syncName: parentSyncName, + isAction: false, + syncType: SyncType.WEBHOOK, + isWebhook: true, + activityLogId, + input, + provider: syncConfig.provider, + debug: false, + temporalContext: context + }); + + const result = await syncRun.run(); + + return result.success; +} + export async function scheduleAndRouteSync(args: ContinuousSyncArgs): Promise { const { syncId, activityLogId, syncName, nangoConnection, debug } = args; let environmentId = nangoConnection?.environment_id; @@ -309,13 +352,24 @@ export async function syncProvider( export async function reportFailure( error: any, - workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs, + workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs | WebhookArgs, DEFAULT_TIMEOUT: string, MAXIMUM_ATTEMPTS: number ): Promise { const { nangoConnection } = workflowArguments; - const type = 'syncName' in workflowArguments ? 'sync' : 'action'; - const name = 'syncName' in workflowArguments ? workflowArguments.syncName : workflowArguments.actionName; + let type = 'webhook'; + + let name = ''; + if ('syncName' in workflowArguments) { + name = workflowArguments.syncName; + type = 'sync'; + } else if ('actionName' in workflowArguments) { + name = workflowArguments.actionName; + type = 'action'; + } else { + name = workflowArguments.name; + } + let content = `The ${type} "${name}" failed `; const context: Context = Context.current(); diff --git a/packages/worker/lib/integration.service.ts b/packages/worker/lib/integration.service.ts index 51267ca277f..98d10a4969c 100644 --- a/packages/worker/lib/integration.service.ts +++ b/packages/worker/lib/integration.service.ts @@ -6,6 +6,7 @@ import { getRootDir, NangoIntegrationData, NangoProps, + NangoAction, NangoSync, localFileService, remoteFileService, @@ -30,13 +31,14 @@ class IntegrationService implements IntegrationServiceInterface { integrationData: NangoIntegrationData, environmentId: number, writeToDb: boolean, - isAction: boolean, + isInvokedImmediately: boolean, + isWebhook: boolean, optionalLoadLocation?: string, input?: object, temporalContext?: Context ): Promise> { try { - const nango = new NangoSync(nangoProps); + const nango = isInvokedImmediately && !isWebhook ? new NangoAction(nangoProps) : new NangoSync(nangoProps); const script: string | null = isCloud() && !optionalLoadLocation ? await remoteFileService.getFile(integrationData.fileLocation as string, environmentId) @@ -87,26 +89,52 @@ class IntegrationService implements IntegrationServiceInterface { const rootDir = getRootDir(optionalLoadLocation); const scriptExports = vm.run(script as string, `${rootDir}/*.js`); - if (typeof scriptExports.default === 'function') { - const results = isAction ? await scriptExports.default(nango, input) : await scriptExports.default(nango); + if (isWebhook) { + if (!scriptExports.onWebhookPayloadReceived) { + const content = `There is no onWebhookPayloadReceived export for ${syncName}`; + if (activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content, + timestamp: Date.now() + }); + } + + return { success: false, error: new NangoError(content, 500), response: null }; + } + + const results = await scriptExports.onWebhookPayloadReceived(nango, input); return { success: true, error: null, response: results }; } else { - const content = `There is no default export that is a function for ${syncName}`; - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content, - timestamp: Date.now() - }); + if (typeof scriptExports.default === 'function') { + const results = isInvokedImmediately ? await scriptExports.default(nango, input) : await scriptExports.default(nango); + + return { success: true, error: null, response: results }; + } else { + const content = `There is no default export that is a function for ${syncName}`; + if (activityLogId && writeToDb) { + await createActivityLogMessage({ + level: 'error', + environment_id: environmentId, + activity_log_id: activityLogId, + content, + timestamp: Date.now() + }); + } + + return { success: false, error: new NangoError(content, 500), response: null }; } - - return { success: false, error: new NangoError(content, 500), response: null }; } } catch (err: any) { - const errorType = isAction ? 'action_script_failure' : 'sync_script_failure'; + let errorType = 'sync_script_failure'; + if (isWebhook) { + errorType = 'webhook_script_failure'; + } else if (isInvokedImmediately) { + errorType = 'action_script_failure'; + } const { success, error, response } = formatScriptError(err, errorType, syncName); if (activityLogId && writeToDb) { diff --git a/packages/worker/lib/models/Worker.ts b/packages/worker/lib/models/Worker.ts index fafa9297ec9..3cae8d7a052 100644 --- a/packages/worker/lib/models/Worker.ts +++ b/packages/worker/lib/models/Worker.ts @@ -24,3 +24,11 @@ export interface ActionArgs { nangoConnection: NangoConnection; activityLogId: number; } + +export interface WebhookArgs { + name: string; + parentSyncName: string; + nangoConnection: NangoConnection; + input: object; + activityLogId: number; +} diff --git a/packages/worker/lib/worker.ts b/packages/worker/lib/worker.ts index fd5d8fe9804..f29d347eb96 100644 --- a/packages/worker/lib/worker.ts +++ b/packages/worker/lib/worker.ts @@ -3,7 +3,7 @@ import fs from 'fs-extra'; import * as dotenv from 'dotenv'; import { createRequire } from 'module'; import * as activities from './activities.js'; -import { TASK_QUEUE, isProd } from '@nangohq/shared'; +import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd } from '@nangohq/shared'; async function run() { if (process.env['SERVER_RUN_MODE'] !== 'DOCKERIZED') { @@ -32,22 +32,27 @@ async function run() { } }); - const worker = await Worker.create({ + const syncWorker = { connection, namespace, workflowsPath: createRequire(import.meta.url).resolve('./workflows'), activities, - taskQueue: TASK_QUEUE, - maxConcurrentWorkflowTaskExecutions: 50 - }); - // Worker connects to localhost by default and uses console.error for logging. - // Customize the Worker by passing more options to create(): - // https://typescript.temporal.io/api/classes/worker.Worker - // If you need to configure server connection parameters, see docs: - // https://docs.temporal.io/typescript/security#encryption-in-transit-with-mtls - - // Step 2: Start accepting tasks on the `${TASK_QUEUE}` queue - await worker.run(); + maxConcurrentWorkflowTaskExecutions: 50, + taskQueue: SYNC_TASK_QUEUE + }; + + const webhookWorker = { + connection, + namespace, + workflowsPath: createRequire(import.meta.url).resolve('./workflows'), + activities, + maxConcurrentWorkflowTaskExecutions: 50, + maxActivitiesPerSecond: 50, + taskQueue: WEBHOOK_TASK_QUEUE + }; + + const workers = await Promise.all([Worker.create(syncWorker), Worker.create(webhookWorker)]); + await Promise.all(workers.map((worker) => worker.run())); } run().catch((err) => { diff --git a/packages/worker/lib/workflows.ts b/packages/worker/lib/workflows.ts index 7681fece7f0..bb0d50c199d 100644 --- a/packages/worker/lib/workflows.ts +++ b/packages/worker/lib/workflows.ts @@ -1,11 +1,11 @@ import { proxyActivities } from '@temporalio/workflow'; import type * as activities from './activities.js'; -import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/Worker'; +import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs, WebhookArgs } from './models/Worker'; const DEFAULT_TIMEOUT = '24 hours'; const MAXIMUM_ATTEMPTS = 3; -const { reportFailure, routeSync, scheduleAndRouteSync, runAction } = proxyActivities({ +const { reportFailure, routeSync, scheduleAndRouteSync, runAction, runWebhook } = proxyActivities({ startToCloseTimeout: DEFAULT_TIMEOUT, scheduleToCloseTimeout: DEFAULT_TIMEOUT, retry: { @@ -44,3 +44,13 @@ export async function action(args: ActionArgs): Promise { return { success: false }; } } + +export async function webhook(args: WebhookArgs): Promise { + try { + return await runWebhook(args); + } catch (e: any) { + await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); + + return false; + } +} From 9b7dfdd0e937e17cc8d4691e5d7e1acf68708eb5 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 15:25:05 -0500 Subject: [PATCH 09/85] 0.36.55 --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 4 ++-- packages/server/package.json | 4 ++-- packages/shared/flows.yaml | 11 +++++++++++ packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- scripts/release.bash | 2 ++ 14 files changed, 51 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index e5fe68ccaaf..8108f712d28 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15301,12 +15301,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.52", + "version": "0.36.55", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15407,7 +15407,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.52", + "version": "0.36.55", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15415,7 +15415,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15678,7 +15678,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.52", + "version": "0.36.55", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15695,7 +15695,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15720,11 +15720,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.52", + "version": "0.36.55", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15783,13 +15783,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.52", + "version": "0.36.55", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.52", + "@nangohq/node": "0.36.55", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16010,9 +16010,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.52", + "version": "0.36.55", "dependencies": { - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 5bcbef397f3..a5e8d84109b 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.52 + image: nangohq/nango-server:0.36.55 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.52 + image: nangohq/nango-worker:0.36.55 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index 8eae996b934..f3e700ff5cf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.52", + "version": "0.36.55", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 7a83a8b10e0..0ee9f293d3a 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.52", + "version": "0.36.55", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 4572342cff0..467b1f6ed32 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 4176c94f1a5..f32450d0874 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.52", + "version": "0.36.55", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index 611d1c16076..0cb223cc87a 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -27,4 +27,4 @@ "@types/node": "^18.7.6", "typescript": "^5.3.2" } -} \ No newline at end of file +} diff --git a/packages/server/package.json b/packages/server/package.json index 87a2e5c20fa..7e9ddfbbd7c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.52", + "version": "0.36.55", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/flows.yaml b/packages/shared/flows.yaml index 4b28f895abb..d08de5d9f3e 100644 --- a/packages/shared/flows.yaml +++ b/packages/shared/flows.yaml @@ -445,6 +445,10 @@ integrations: runs: every half hour returns: - HubspotServiceTicket + hubspot-contacts: + runs: every day + returns: + - HubspotContact hubspot-owner: runs: every day returns: @@ -493,6 +497,13 @@ integrations: category: string content: string publishDate: number + HubspotContact: + id: string + created_at: string + updated_at: string + first_name: string + last_name: string + email: string intercom: intercom-conversations: runs: every 6 hours diff --git a/packages/shared/package.json b/packages/shared/package.json index 7fab39d4310..df55cac70c6 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.52", + "version": "0.36.55", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.52", + "@nangohq/node": "0.36.55", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index ad29aad52ca..82b6d028d52 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.52", + "version": "0.36.55", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.52", + "version": "0.36.55", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.52", + "@nangohq/frontend": "0.36.55", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.52", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.52.tgz", - "integrity": "sha512-LPv5c7bOf779grT9fyXJizOZSDRkHTtvedgp+1DA7wtlIYHcHY2Jf/AR7oR/SZnFP7NaO36AueEjeJAw7PdZbw==" + "version": "0.36.55", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.55.tgz", + "integrity": "sha512-sVRbLbHJmp2A2Ofqu9/bMxgWYfVMXrZooK410E5puNLICfQj0Sr+Hz6GgRcs/1BnRTyVNDuLWDq1zTBxbjgH1A==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.52", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.52.tgz", - "integrity": "sha512-LPv5c7bOf779grT9fyXJizOZSDRkHTtvedgp+1DA7wtlIYHcHY2Jf/AR7oR/SZnFP7NaO36AueEjeJAw7PdZbw==" + "version": "0.36.55", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.55.tgz", + "integrity": "sha512-sVRbLbHJmp2A2Ofqu9/bMxgWYfVMXrZooK410E5puNLICfQj0Sr+Hz6GgRcs/1BnRTyVNDuLWDq1zTBxbjgH1A==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 3abc84cdedf..1f6866abdf3 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.52", + "version": "0.36.55", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.52", + "@nangohq/frontend": "0.36.55", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 49ac210d2c8..4b9399e6c1d 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.52", + "version": "0.36.55", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.52", + "@nangohq/shared": "0.36.55", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/scripts/release.bash b/scripts/release.bash index 16e17ce9e00..f074c472a72 100755 --- a/scripts/release.bash +++ b/scripts/release.bash @@ -99,6 +99,8 @@ cd ./packages/shared && npm publish --access public && cd ../../ update_shared_dep "packages/server/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) update_shared_dep "packages/worker/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) update_shared_dep "packages/cli/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) +update_shared_dep "packages/jobs/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) +update_shared_dep "packages/runner/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) # update the webapp and frontend FRONTEND_PACKAGE_JSON="packages/frontend/package.json" From bc2d2cbf7915ac3376718e9cb007922954b2d899 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 15:41:52 -0500 Subject: [PATCH 10/85] allow to push to jobs or runner --- scripts/docker-publish.bash | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/scripts/docker-publish.bash b/scripts/docker-publish.bash index d498f64aedc..2bc9e3d6145 100755 --- a/scripts/docker-publish.bash +++ b/scripts/docker-publish.bash @@ -7,21 +7,30 @@ ENV=$4 PACKAGE_NAME=${package:6} -# allow for custom worker cloud name -if [ "$ENV" == "staging" ]; then - npm run build:staging && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/nango-cloud-staging:$version -t nangohq/nango-cloud-staging:latest . --no-cache --output type=registry -fi -if [ "$ENV" == "prod" ]; then - npm run build:prod && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/nango-cloud:$version -t nangohq/nango-cloud:latest . --no-cache --output type=registry -fi +if [ "$PACKAGE_NAME" == "jobs" ] || [ "$PACKAGE_NAME" == "runner" ]; then + if [ "$ENV" == "staging" ]; then + npm run ts-build && docker build -f packages/$PACKAGE_NAME/Dockerfile --platform linux/amd64 -t nangohq/$package:$(git rev-parse --short HEAD) -t nangohq/$package:staging . && docker push nangohq/$package --all-tags + fi -if [ "$ENV" == "hosted" ]; then - if [ "$tagLatest" == "true" ]; then - npm run build:hosted && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/$package:$version -t nangohq/$package:latest . --no-cache --output type=registry - else - npm run build:hosted && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/$package:$version . --no-cache --output type=registry + if [ "$ENV" == "prod" ]; then + npm run ts-build && docker build -f packages/$PACKAGE_NAME/Dockerfile --platform linux/amd64 -t nangohq/$package:$(git rev-parse --short HEAD) -t nangohq/$package:production . && docker push nangohq/$package --all-tags + fi +else + # allow for custom worker cloud name + if [ "$ENV" == "staging" ]; then + npm run build:staging && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/nango-cloud-staging:$version -t nangohq/nango-cloud-staging:latest . --no-cache --output type=registry fi -fi + if [ "$ENV" == "prod" ]; then + npm run build:prod && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/nango-cloud:$version -t nangohq/nango-cloud:latest . --no-cache --output type=registry + fi + if [ "$ENV" == "hosted" ]; then + if [ "$tagLatest" == "true" ]; then + npm run build:hosted && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/$package:$version -t nangohq/$package:latest . --no-cache --output type=registry + else + npm run build:hosted && docker buildx build --platform linux/amd64 -f packages/$PACKAGE_NAME/Dockerfile -t nangohq/$package:$version . --no-cache --output type=registry + fi + fi +fi From e4a43d6b6a1fde3f653294bdb789081f5a093f72 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 16:37:13 -0500 Subject: [PATCH 11/85] [nan-20] webhook additions --- packages/server/lib/controllers/webhook.controller.ts | 3 +++ .../integrations/scripts/webhook/hubspot-webhook-routing.ts | 1 + 2 files changed, 4 insertions(+) diff --git a/packages/server/lib/controllers/webhook.controller.ts b/packages/server/lib/controllers/webhook.controller.ts index f9a4ab69332..01dfe7c07bc 100644 --- a/packages/server/lib/controllers/webhook.controller.ts +++ b/packages/server/lib/controllers/webhook.controller.ts @@ -21,7 +21,10 @@ class WebhookController { if (areWebhooksEnabled) { routeWebhook(environmentUuid, providerConfigKey, headers, req.body); + } else { + res.status(404).send(); } + res.status(200).send(); } catch (err) { next(err); diff --git a/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts b/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts index 8dea8921c73..94d4a093db5 100644 --- a/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts +++ b/packages/shared/lib/integrations/scripts/webhook/hubspot-webhook-routing.ts @@ -15,6 +15,7 @@ export default async function route(nango: Nango, integration: ProviderConfig, h const valid = validate(integration, headers, body); if (!valid) { + console.log('Hubspot webhook signature invalid'); return; } From f67eab9a362fbbefa7a28e3d6d1b9b480822d994 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 16:43:57 -0500 Subject: [PATCH 12/85] [nan-20] enabled by default --- packages/server/lib/controllers/webhook.controller.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/lib/controllers/webhook.controller.ts b/packages/server/lib/controllers/webhook.controller.ts index 01dfe7c07bc..b739f0217ed 100644 --- a/packages/server/lib/controllers/webhook.controller.ts +++ b/packages/server/lib/controllers/webhook.controller.ts @@ -17,7 +17,7 @@ class WebhookController { return; } - const areWebhooksEnabled = await featureFlags.isEnabled('external-webhooks', accountUUID, true); + const areWebhooksEnabled = await featureFlags.isEnabled('external-webhooks', accountUUID, true, true); if (areWebhooksEnabled) { routeWebhook(environmentUuid, providerConfigKey, headers, req.body); From b6450d856946afa51c32b69e00df5a16b1e0c898 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 16:49:01 -0500 Subject: [PATCH 13/85] 0.36.56 --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8108f712d28..b16a8e30f18 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15301,12 +15301,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.55", + "version": "0.36.56", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15407,7 +15407,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.55", + "version": "0.36.56", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15415,7 +15415,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15678,7 +15678,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.55", + "version": "0.36.56", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15695,7 +15695,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15720,11 +15720,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.55", + "version": "0.36.56", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15783,13 +15783,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.55", + "version": "0.36.56", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.55", + "@nangohq/node": "0.36.56", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16010,9 +16010,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.55", + "version": "0.36.56", "dependencies": { - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index a5e8d84109b..50d992b847a 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.55 + image: nangohq/nango-server:0.36.56 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.55 + image: nangohq/nango-worker:0.36.56 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index f3e700ff5cf..094c4c3830e 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.55", + "version": "0.36.56", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 0ee9f293d3a..cdc7dc9c282 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.55", + "version": "0.36.56", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 467b1f6ed32..d46687e1421 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index f32450d0874..91703b442a3 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.55", + "version": "0.36.56", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index 0cb223cc87a..2747acf98d8 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index 7e9ddfbbd7c..833aeedf8b6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.55", + "version": "0.36.56", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index df55cac70c6..ab6d6c1c187 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.55", + "version": "0.36.56", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.55", + "@nangohq/node": "0.36.56", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index 82b6d028d52..4b102028d35 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.55", + "version": "0.36.56", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.55", + "version": "0.36.56", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.55", + "@nangohq/frontend": "0.36.56", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.55", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.55.tgz", - "integrity": "sha512-sVRbLbHJmp2A2Ofqu9/bMxgWYfVMXrZooK410E5puNLICfQj0Sr+Hz6GgRcs/1BnRTyVNDuLWDq1zTBxbjgH1A==" + "version": "0.36.56", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.56.tgz", + "integrity": "sha512-VC7rHOdkrDGd32xbKo08pWGPpnwqJ3LzlY46rcQJe+3it10Fc5TNE5VFbcxurmwxJSvg+cfbQNwHLuuW30Zztw==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.55", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.55.tgz", - "integrity": "sha512-sVRbLbHJmp2A2Ofqu9/bMxgWYfVMXrZooK410E5puNLICfQj0Sr+Hz6GgRcs/1BnRTyVNDuLWDq1zTBxbjgH1A==" + "version": "0.36.56", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.56.tgz", + "integrity": "sha512-VC7rHOdkrDGd32xbKo08pWGPpnwqJ3LzlY46rcQJe+3it10Fc5TNE5VFbcxurmwxJSvg+cfbQNwHLuuW30Zztw==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 1f6866abdf3..95cb0e27a20 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.55", + "version": "0.36.56", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.55", + "@nangohq/frontend": "0.36.56", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 4b9399e6c1d..34b0ab5ec6c 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.55", + "version": "0.36.56", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.55", + "@nangohq/shared": "0.36.56", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From d7bb554d30167c545eb110d7a2854504975c0256 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 17:20:52 -0500 Subject: [PATCH 14/85] remove jobs migration for now --- .../20231208181152_update_jobs_type_enum.cjs | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs index 53e22968e46..ae2ae45c67b 100644 --- a/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs +++ b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs @@ -1,31 +1,4 @@ const tableName = '_nango_sync_jobs'; -exports.up = function(knex, _) { - return knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.string('type_new').defaultsTo('initial').notNullable(); - }) - .then(() => knex.raw(` - UPDATE nango.${tableName} SET type_new = type; - `)) - .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.dropColumn('type'); - })) - .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.renameColumn('type_new', 'type'); - })); -}; - -exports.down = function(knex, _) { - return knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.enu('type', ['INITIAL', 'INCREMENTAL']).defaultTo('initial').notNullable(); - }) - .then(() => knex.raw(` - UPDATE nango.${tableName} SET type_new = type; - `)) - .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.dropColumn('type'); - })) - .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { - table.renameColumn('type_new', 'type'); - })); -}; +exports.up = function(knex) { }; +exports.down = function(knex) { }; From 3c1e0e17ff2bbb7511b7cd4ad6220285cd98f869 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 18 Dec 2023 17:26:08 -0500 Subject: [PATCH 15/85] [nan-20] bring back type adjustment --- .../20231208181152_update_jobs_type_enum.cjs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs index ae2ae45c67b..53e22968e46 100644 --- a/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs +++ b/packages/shared/lib/db/migrations/20231208181152_update_jobs_type_enum.cjs @@ -1,4 +1,31 @@ const tableName = '_nango_sync_jobs'; -exports.up = function(knex) { }; -exports.down = function(knex) { }; +exports.up = function(knex, _) { + return knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.string('type_new').defaultsTo('initial').notNullable(); + }) + .then(() => knex.raw(` + UPDATE nango.${tableName} SET type_new = type; + `)) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.dropColumn('type'); + })) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.renameColumn('type_new', 'type'); + })); +}; + +exports.down = function(knex, _) { + return knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.enu('type', ['INITIAL', 'INCREMENTAL']).defaultTo('initial').notNullable(); + }) + .then(() => knex.raw(` + UPDATE nango.${tableName} SET type_new = type; + `)) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.dropColumn('type'); + })) + .then(() => knex.schema.withSchema('nango').alterTable(tableName, function(table) { + table.renameColumn('type_new', 'type'); + })); +}; From dbd096469044aae92e726ae0fdee53ac0292bedd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 18:10:18 -0500 Subject: [PATCH 16/85] Bump vite from 4.4.3 to 4.5.1 (#1396) Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.4.3 to 4.5.1. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.1/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.1/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- package-lock.json | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index b16a8e30f18..b23ad7350ab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12614,9 +12614,10 @@ } }, "node_modules/rollup": { - "version": "3.26.2", + "version": "3.29.4", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.4.tgz", + "integrity": "sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw==", "dev": true, - "license": "MIT", "bin": { "rollup": "dist/bin/rollup" }, @@ -14312,13 +14313,14 @@ } }, "node_modules/vite": { - "version": "4.4.3", + "version": "4.5.1", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.1.tgz", + "integrity": "sha512-AXXFaAJ8yebyqzoNB9fu2pHoo/nWX+xZlaRwoeYUxEqBO+Zj4msE5G+BhGBll9lYEKv9Hfks52PAF2X7qDYXQA==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.18.10", - "postcss": "^8.4.25", - "rollup": "^3.25.2" + "postcss": "^8.4.27", + "rollup": "^3.27.1" }, "bin": { "vite": "bin/vite.js" From 06be47ed85489e688637e17fad4c82e8302fe3be Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 06:56:57 -0500 Subject: [PATCH 17/85] [nan-134] rumi fix nangoyaml generation (#1435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [nan-20] parse new properties and set validation logic • add tests to support new properties * [nan-20] replaceMetada (set) and updateMetada • update docs • adds tests * use connectionCreatedHook to make it easier to manage connection lifecycle * [nan-20] progress on webhook and post connection * [nan-20] webhook logic written in the sync script * [nan-20] allow a webhook update to send a nango webhook once finished * [nan-20] feedback updates * [nan-20] fix spacing * [nan-20] update test failure * [nan-20] remove todo * [nan-20] don't allow invalid webhooks through * [nan-20] query for the connection using the identifier instead of looping over all connections * [nan-20] observability * [nan-20] add jira webhook handling * [nan-20] pr feedback * [nan-134] don't allow commas or semi colons to end * [nan-20] remove deprecated * [nan-20] remove runner node_modules and dist * [nan-20] update webhook logic * [nan-20] add possibility for customer defined feature flag * [nan-20] remove tsbuild file --- nango.yaml | 25 +++++++++++++++++++ .../nango-yaml/v1/no-commas/nango.yaml | 25 +++++++++++++++++++ .../nango-yaml/v1/no-semi-colons/nango.yaml | 25 +++++++++++++++++++ .../nango-yaml/v1/{ => valid}/nango.yaml | 0 .../nango-yaml/v1/{ => valid}/object.json | 0 packages/cli/lib/services/model.service.ts | 10 +++++--- packages/cli/lib/sync.unit.test.ts | 16 ++++++++++-- 7 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 nango.yaml create mode 100644 packages/cli/lib/fixtures/nango-yaml/v1/no-commas/nango.yaml create mode 100644 packages/cli/lib/fixtures/nango-yaml/v1/no-semi-colons/nango.yaml rename packages/cli/lib/fixtures/nango-yaml/v1/{ => valid}/nango.yaml (100%) rename packages/cli/lib/fixtures/nango-yaml/v1/{ => valid}/object.json (100%) diff --git a/nango.yaml b/nango.yaml new file mode 100644 index 00000000000..5ec30203b8f --- /dev/null +++ b/nango.yaml @@ -0,0 +1,25 @@ +integrations: + demo-github-integration: + github-issue-example: + type: sync + runs: every half hour + auto_start: true + returns: + - GithubIssue + github-write-action: + type: action + returns: boolean + +models: + GithubIssue: + id: integer; + owner: string + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string diff --git a/packages/cli/lib/fixtures/nango-yaml/v1/no-commas/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v1/no-commas/nango.yaml new file mode 100644 index 00000000000..c14e12e9bd1 --- /dev/null +++ b/packages/cli/lib/fixtures/nango-yaml/v1/no-commas/nango.yaml @@ -0,0 +1,25 @@ +integrations: + demo-github-integration: + github-issue-example: + type: sync + runs: every half hour + auto_start: true + returns: + - GithubIssue + github-write-action: + type: action + returns: boolean + +models: + GithubIssue: + id: integer, + owner: string, + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string diff --git a/packages/cli/lib/fixtures/nango-yaml/v1/no-semi-colons/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v1/no-semi-colons/nango.yaml new file mode 100644 index 00000000000..5ec30203b8f --- /dev/null +++ b/packages/cli/lib/fixtures/nango-yaml/v1/no-semi-colons/nango.yaml @@ -0,0 +1,25 @@ +integrations: + demo-github-integration: + github-issue-example: + type: sync + runs: every half hour + auto_start: true + returns: + - GithubIssue + github-write-action: + type: action + returns: boolean + +models: + GithubIssue: + id: integer; + owner: string + repo: string + issue_number: number + title: string + author: string + author_id: string + state: string + date_created: date + date_last_modified: date + body: string diff --git a/packages/cli/lib/fixtures/nango-yaml/v1/nango.yaml b/packages/cli/lib/fixtures/nango-yaml/v1/valid/nango.yaml similarity index 100% rename from packages/cli/lib/fixtures/nango-yaml/v1/nango.yaml rename to packages/cli/lib/fixtures/nango-yaml/v1/valid/nango.yaml diff --git a/packages/cli/lib/fixtures/nango-yaml/v1/object.json b/packages/cli/lib/fixtures/nango-yaml/v1/valid/object.json similarity index 100% rename from packages/cli/lib/fixtures/nango-yaml/v1/object.json rename to packages/cli/lib/fixtures/nango-yaml/v1/valid/object.json diff --git a/packages/cli/lib/services/model.service.ts b/packages/cli/lib/services/model.service.ts index 164b212c84c..31f0e6536f0 100644 --- a/packages/cli/lib/services/model.service.ts +++ b/packages/cli/lib/services/model.service.ts @@ -69,7 +69,7 @@ class ModelService { }) .map((fieldName: string) => { const fieldModel = fields[fieldName] as string | NangoModel; - const fieldType = this.getFieldType(fieldModel, debug); + const fieldType = this.getFieldType(fieldModel, debug, modelName); return ` ${fieldName}: ${fieldType};`; }) .join('\n'); @@ -80,8 +80,12 @@ class ModelService { return interfaceDefinitions; } - private getFieldType(rawField: string | NangoModel, debug = false): string { + private getFieldType(rawField: string | NangoModel, debug = false, modelName: string): string { if (typeof rawField === 'string') { + if (rawField.toString().endsWith(',') || rawField.toString().endsWith(';')) { + throw new Error(`Field "${rawField}" in the model ${modelName} ends with a comma or semicolon which is not allowed.`); + } + let field = rawField; let hasNull = false; let hasUndefined = false; @@ -137,7 +141,7 @@ class ModelService { } else { try { const nestedFields = Object.keys(rawField) - .map((fieldName: string) => ` ${fieldName}: ${this.getFieldType(rawField[fieldName] as string | NangoModel)};`) + .map((fieldName: string) => ` ${fieldName}: ${this.getFieldType(rawField[fieldName] as string | NangoModel, debug, modelName)};`) .join('\n'); return `{\n${nestedFields}\n}`; } catch (_) { diff --git a/packages/cli/lib/sync.unit.test.ts b/packages/cli/lib/sync.unit.test.ts index cd13b155171..f738dfd5c9e 100644 --- a/packages/cli/lib/sync.unit.test.ts +++ b/packages/cli/lib/sync.unit.test.ts @@ -313,12 +313,24 @@ describe('generate function tests', () => { }); it('should parse a nango.yaml file that is version 1 as expected', async () => { - const { response: config } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v1`)); + const { response: config } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v1/valid`)); expect(config).toBeDefined(); - const json = fs.readFileSync(path.resolve(__dirname, `./fixtures/nango-yaml/v1/object.json`), 'utf8'); + const json = fs.readFileSync(path.resolve(__dirname, `./fixtures/nango-yaml/v1/valid/object.json`), 'utf8'); expect(config).toEqual(JSON.parse(json)); }); + it('v1 - should complain about commas at the end of declared types', async () => { + await fs.promises.mkdir(testDirectory, { recursive: true }); + await fs.promises.copyFile(`${fixturesPath}/nango-yaml/v1/no-commas/nango.yaml`, `${testDirectory}/nango.yaml`); + expect(generate(false, true)).rejects.toThrow(`Field "integer," in the model GithubIssue ends with a comma or semicolon which is not allowed.`); + }); + + it('v1 - should complain about semi colons at the end of declared types', async () => { + await fs.promises.mkdir(testDirectory, { recursive: true }); + await fs.promises.copyFile(`${fixturesPath}/nango-yaml/v1/no-semi-colons/nango.yaml`, `${testDirectory}/nango.yaml`); + expect(generate(false, true)).rejects.toThrow(`Field "integer;" in the model GithubIssue ends with a comma or semicolon which is not allowed.`); + }); + it('should parse a nango.yaml file that is version 2 as expected', async () => { const { response: config } = await configService.load(path.resolve(__dirname, `./fixtures/nango-yaml/v2/valid`)); expect(config).toBeDefined(); From 311506ba936396bd0e75fe83510f397c5219a41a Mon Sep 17 00:00:00 2001 From: Bastien Beurier Date: Tue, 19 Dec 2023 16:05:36 +0100 Subject: [PATCH 18/85] Add Intercom docs details --- docs-v2/integrations/all/intercom.mdx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs-v2/integrations/all/intercom.mdx b/docs-v2/integrations/all/intercom.mdx index e7df7bae19f..72a0d208535 100644 --- a/docs-v2/integrations/all/intercom.mdx +++ b/docs-v2/integrations/all/intercom.mdx @@ -19,6 +19,7 @@ API configuration: [`intercom`](https://nango.dev/providers.yaml) ## Getting started +- [Create a Developer Workspace](https://app.intercom.com/a/developer-signup) - [How to register an Application](https://developers.intercom.com/building-apps/docs/setting-up-oauth) - [OAuth-related docs](https://developers.intercom.com/building-apps/docs/setting-up-oauth) - [List of OAuth scopes](https://developers.intercom.com/building-apps/docs/oauth-scopes) @@ -28,4 +29,7 @@ API configuration: [`intercom`](https://nango.dev/providers.yaml) ## API gotchas +- Intercom access tokens do not expire. Logically, Intercom doesn't return a refresh token along with the access token. +- You do not need to pass API scopes/permissions during the authorization request. Permissions are only set in the Intercom Developer Portal. + Add Getting Started links and Gotchas by [editing this page](https://github.com/nangohq/nango/tree/master/docs-v2/integrations/all/instagram.mdx) From eed340a857d80a0fd8fc8c75897c91f7aa759765 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:07:21 +0100 Subject: [PATCH 19/85] Add 60s default timeout to db client (#1434) * Add 60s default timeout to db client * delete old activity logs every hour * no timeout for db migrations * remove 'drop index created_at_index' migration This index didn't actually exist --- packages/server/lib/jobs/index.ts | 2 +- packages/server/lib/utils/migrate.ts | 3 ++- packages/shared/lib/db/config.ts | 27 ------------------- packages/shared/lib/db/database.ts | 26 +++++++++++++++--- ...3124728_drop_sync_data_records_indexes.cjs | 1 - 5 files changed, 25 insertions(+), 34 deletions(-) delete mode 100644 packages/shared/lib/db/config.ts diff --git a/packages/server/lib/jobs/index.ts b/packages/server/lib/jobs/index.ts index 8180aabd3e6..c41b14e9361 100644 --- a/packages/server/lib/jobs/index.ts +++ b/packages/server/lib/jobs/index.ts @@ -6,7 +6,7 @@ export async function deleteOldActivityLogs(): Promise { /** * Delete all activity logs older than 15 days */ - cron.schedule('0 0 * * *', async () => { + cron.schedule('0 * * * *', async () => { const activityLogTableName = '_nango_activity_logs'; await db.knex.withSchema(db.schema()).from(activityLogTableName).where('created_at', '<', db.knex.raw("now() - interval '15 days'")).del(); }); diff --git a/packages/server/lib/utils/migrate.ts b/packages/server/lib/utils/migrate.ts index 8cce1e96220..51f86fb6be2 100644 --- a/packages/server/lib/utils/migrate.ts +++ b/packages/server/lib/utils/migrate.ts @@ -1,7 +1,8 @@ import Logger from '../utils/logger.js'; -import { db, encryptionManager } from '@nangohq/shared'; +import { encryptionManager, KnexDatabase } from '@nangohq/shared'; export default async function migrate() { + const db = new KnexDatabase({ timeoutMs: 0 }); // Disable timeout for migrations Logger.info('Migrating database ...'); await db.knex.raw(`CREATE SCHEMA IF NOT EXISTS ${db.schema()}`); await db.migrate(process.env['NANGO_DB_MIGRATION_FOLDER'] || '../shared/lib/db/migrations'); diff --git a/packages/shared/lib/db/config.ts b/packages/shared/lib/db/config.ts deleted file mode 100644 index 5c8c44c3fae..00000000000 --- a/packages/shared/lib/db/config.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { Knex } from 'knex'; - -const config: { development: Knex.Config; production: Knex.Config } = { - development: { - client: process.env['NANGO_DB_CLIENT'] || 'pg', - connection: process.env['NANGO_DATABASE_URL'] || { - host: process.env['NANGO_DB_HOST'] || (process.env['SERVER_RUN_MODE'] === 'DOCKERIZED' ? 'nango-db' : 'localhost'), - port: +(process.env['NANGO_DB_PORT'] || 5432), - user: process.env['NANGO_DB_USER'] || 'nango', - database: process.env['NANGO_DB_NAME'] || 'nango', - password: process.env['NANGO_DB_PASSWORD'] || 'nango', - ssl: process.env['NANGO_DB_SSL'] != null && process.env['NANGO_DB_SSL'].toLowerCase() === 'true' ? { rejectUnauthorized: false } : undefined - }, - migrations: { - directory: './migrations', - extension: 'ts' - }, - pool: { - min: parseInt(process.env['NANGO_DB_POOL_MIN'] || '2'), - max: parseInt(process.env['NANGO_DB_POOL_MAX'] || '7') - } - }, - - production: {} -}; - -export { config }; diff --git a/packages/shared/lib/db/database.ts b/packages/shared/lib/db/database.ts index f616c5a3d66..1bdca15d72a 100644 --- a/packages/shared/lib/db/database.ts +++ b/packages/shared/lib/db/database.ts @@ -1,12 +1,30 @@ import knex from 'knex'; import type { Knex } from 'knex'; -import { config } from './config.js'; -class KnexDatabase { +function getDbConfig({ timeoutMs }: { timeoutMs: number }): Knex.Config { + return { + client: process.env['NANGO_DB_CLIENT'] || 'pg', + connection: process.env['NANGO_DATABASE_URL'] || { + host: process.env['NANGO_DB_HOST'] || (process.env['SERVER_RUN_MODE'] === 'DOCKERIZED' ? 'nango-db' : 'localhost'), + port: +(process.env['NANGO_DB_PORT'] || 5432), + user: process.env['NANGO_DB_USER'] || 'nango', + database: process.env['NANGO_DB_NAME'] || 'nango', + password: process.env['NANGO_DB_PASSWORD'] || 'nango', + ssl: process.env['NANGO_DB_SSL'] != null && process.env['NANGO_DB_SSL'].toLowerCase() === 'true' ? { rejectUnauthorized: false } : undefined, + statement_timeout: timeoutMs + }, + pool: { + min: parseInt(process.env['NANGO_DB_POOL_MIN'] || '2'), + max: parseInt(process.env['NANGO_DB_POOL_MAX'] || '7') + } + }; +} + +export class KnexDatabase { knex: Knex; - constructor() { - const dbConfig = config.development; + constructor({ timeoutMs } = { timeoutMs: 60000 }) { + const dbConfig = getDbConfig({ timeoutMs }); this.knex = knex(dbConfig); } diff --git a/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs index 1b6de2e0da8..c208f9a2e52 100644 --- a/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs +++ b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs @@ -2,7 +2,6 @@ exports.config = { transaction: false }; exports.up = function(knex) { return knex.schema - .raw('DROP INDEX CONCURRENTLY created_at_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_data_hash_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_sync_job_id_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_external_is_deleted_index') From 7a3c8f87910ce320063f7e6140e753666bfe5028 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:37:45 +0100 Subject: [PATCH 20/85] 0.36.57 (#1444) --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index b23ad7350ab..b41445a1ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15303,12 +15303,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.56", + "version": "0.36.57", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15409,7 +15409,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.56", + "version": "0.36.57", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15417,7 +15417,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15680,7 +15680,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.56", + "version": "0.36.57", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15697,7 +15697,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15722,11 +15722,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.56", + "version": "0.36.57", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15785,13 +15785,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.56", + "version": "0.36.57", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.56", + "@nangohq/node": "0.36.57", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16012,9 +16012,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.56", + "version": "0.36.57", "dependencies": { - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 50d992b847a..4e3914f5852 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.56 + image: nangohq/nango-server:0.36.57 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.56 + image: nangohq/nango-worker:0.36.57 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index 094c4c3830e..4bbf2834620 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.56", + "version": "0.36.57", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index cdc7dc9c282..a2e9a1edd8d 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.56", + "version": "0.36.57", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index d46687e1421..2faa5759f4e 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 91703b442a3..8d5aee14861 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.56", + "version": "0.36.57", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index 2747acf98d8..c0a6ffabc99 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index 833aeedf8b6..d2c69df427a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.56", + "version": "0.36.57", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index ab6d6c1c187..22103ace465 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.56", + "version": "0.36.57", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.56", + "@nangohq/node": "0.36.57", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index 4b102028d35..fece1274aab 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.56", + "version": "0.36.57", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.56", + "version": "0.36.57", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.56", + "@nangohq/frontend": "0.36.57", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.56", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.56.tgz", - "integrity": "sha512-VC7rHOdkrDGd32xbKo08pWGPpnwqJ3LzlY46rcQJe+3it10Fc5TNE5VFbcxurmwxJSvg+cfbQNwHLuuW30Zztw==" + "version": "0.36.57", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.57.tgz", + "integrity": "sha512-hf2ySeTDz5noWcO2qf9mKaWjv0xSJfmNgzohzg38GVa3zGNdLkMTN4zpxq4cpZ+g6t0dTsiuNkSL9Uw63HSeiA==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.56", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.56.tgz", - "integrity": "sha512-VC7rHOdkrDGd32xbKo08pWGPpnwqJ3LzlY46rcQJe+3it10Fc5TNE5VFbcxurmwxJSvg+cfbQNwHLuuW30Zztw==" + "version": "0.36.57", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.57.tgz", + "integrity": "sha512-hf2ySeTDz5noWcO2qf9mKaWjv0xSJfmNgzohzg38GVa3zGNdLkMTN4zpxq4cpZ+g6t0dTsiuNkSL9Uw63HSeiA==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 95cb0e27a20..ad5367f4720 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.56", + "version": "0.36.57", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.56", + "@nangohq/frontend": "0.36.57", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 34b0ab5ec6c..10648b5200d 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.56", + "version": "0.36.57", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.56", + "@nangohq/shared": "0.36.57", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From c49abd11e9bf360326f38c774fefa79d1a53ce03 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:41:15 +0100 Subject: [PATCH 21/85] Runner: Buffer should be available inside Node.vm (#1443) --- packages/runner/lib/exec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/runner/lib/exec.ts b/packages/runner/lib/exec.ts index 78e37471912..4014d416a76 100644 --- a/packages/runner/lib/exec.ts +++ b/packages/runner/lib/exec.ts @@ -1,5 +1,6 @@ import type { NangoProps } from '@nangohq/shared'; import { NangoSync, NangoAction } from '@nangohq/shared'; +import { Buffer } from 'buffer'; import * as vm from 'vm'; import * as url from 'url'; import * as crypto from 'crypto'; @@ -29,7 +30,8 @@ export async function exec(nangoProps: NangoProps, isInvokedImmediately: boolean default: throw new Error(`Module '${moduleName}' is not allowed`); } - } + }, + Buffer }; const context = vm.createContext(sandbox); const scriptExports = script.runInContext(context); From 964e477efbde2993bf8924eb5b40f8fd8e832a91 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:57:35 +0100 Subject: [PATCH 22/85] 0.36.58 (#1445) --- package-lock.json | 22 +++++++++++----------- packages/cli/package.json | 2 +- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/package-lock.json b/package-lock.json index b41445a1ca0..4334d9149eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15308,7 +15308,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15409,7 +15409,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.57", + "version": "0.36.58", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15417,7 +15417,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15680,7 +15680,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.57", + "version": "0.36.58", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15697,7 +15697,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15722,11 +15722,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.57", + "version": "0.36.58", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15785,13 +15785,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.57", + "version": "0.36.58", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.57", + "@nangohq/node": "0.36.58", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16012,9 +16012,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.57", + "version": "0.36.58", "dependencies": { - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4bbf2834620..590419edc28 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index a2e9a1edd8d..1dc477b8926 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.57", + "version": "0.36.58", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 2faa5759f4e..ace7bf913b2 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 8d5aee14861..3a6c768a475 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.57", + "version": "0.36.58", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index c0a6ffabc99..1aa073c4109 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index d2c69df427a..c1e364fd1d2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.57", + "version": "0.36.58", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index 22103ace465..d200a922fe3 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.57", + "version": "0.36.58", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.57", + "@nangohq/node": "0.36.58", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index fece1274aab..a3abc789f74 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.57", + "version": "0.36.58", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.57", + "version": "0.36.58", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.57", + "@nangohq/frontend": "0.36.58", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.57", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.57.tgz", - "integrity": "sha512-hf2ySeTDz5noWcO2qf9mKaWjv0xSJfmNgzohzg38GVa3zGNdLkMTN4zpxq4cpZ+g6t0dTsiuNkSL9Uw63HSeiA==" + "version": "0.36.58", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.58.tgz", + "integrity": "sha512-fk+kC7ydjm4blSrwbac65iBD8Sj1Ntcuvs4l1W8Hf3e0+/w15WyPxWo6Yy4riL2oi1+U9b0NXYujWXEU+Edlqg==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.57", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.57.tgz", - "integrity": "sha512-hf2ySeTDz5noWcO2qf9mKaWjv0xSJfmNgzohzg38GVa3zGNdLkMTN4zpxq4cpZ+g6t0dTsiuNkSL9Uw63HSeiA==" + "version": "0.36.58", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.58.tgz", + "integrity": "sha512-fk+kC7ydjm4blSrwbac65iBD8Sj1Ntcuvs4l1W8Hf3e0+/w15WyPxWo6Yy4riL2oi1+U9b0NXYujWXEU+Edlqg==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index ad5367f4720..fd78f04aaf9 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.57", + "version": "0.36.58", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.57", + "@nangohq/frontend": "0.36.58", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 10648b5200d..010b7b2b7d4 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.57", + "version": "0.36.58", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.57", + "@nangohq/shared": "0.36.58", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From 375aca2b26d7c596694101fdf449cf0b06ae080a Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 12:41:05 -0500 Subject: [PATCH 23/85] comment out migration that is causing issues (#1446) * comment out migration that is causing issues * return promise --- .../20231213124728_drop_sync_data_records_indexes.cjs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs index c208f9a2e52..a83a9c731f3 100644 --- a/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs +++ b/packages/shared/lib/db/migrations/20231213124728_drop_sync_data_records_indexes.cjs @@ -1,6 +1,9 @@ exports.config = { transaction: false }; exports.up = function(knex) { + return Promise.resolve(); + /* + * Production only migration return knex.schema .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_data_hash_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_sync_job_id_index') @@ -11,6 +14,9 @@ exports.up = function(knex) { .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_external_is_deleted_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_data_records_deletes_external_deleted_at_index') .raw('DROP INDEX CONCURRENTLY _nango_sync_jobs_deleted_index'); + */ }; -exports.down = function(knex) { }; +exports.down = function(knex) { + return Promise.resolve(); +}; From 3c2f36a2c643f99abe6e8f120d9eaca65a6a6cb5 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 12:48:44 -0500 Subject: [PATCH 24/85] [nan-172] remove sync/records references (#1437) * [nan-172] remove sync/records references * [nan-172] remove dupe line --- docs-v2/guides/sync.mdx | 4 ++-- packages/webapp/src/pages/GettingStarted.tsx | 2 +- packages/webapp/src/utils/language-snippets.tsx | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs-v2/guides/sync.mdx b/docs-v2/guides/sync.mdx index 2f813f601e2..9a159311170 100644 --- a/docs-v2/guides/sync.mdx +++ b/docs-v2/guides/sync.mdx @@ -53,7 +53,7 @@ Now, collect the continually updated data: ```bash curl --request GET \ - --url 'https://api.nango.dev/sync/records?model=' \ + --url 'https://api.nango.dev/records?model=' \ --header 'Authorization: Bearer ' \ --header 'Connection-Id: ' \ --header 'Provider-Config-Key: ' @@ -86,4 +86,4 @@ Dive deeper with the [Customize Syncs & Actions guide](/guides/custom). Explore ## Questions, problems, feedback? -We're here to help! Please reach out on the [Slack community](https://nango.dev/slack), we are very active there. \ No newline at end of file +We're here to help! Please reach out on the [Slack community](https://nango.dev/slack), we are very active there. diff --git a/packages/webapp/src/pages/GettingStarted.tsx b/packages/webapp/src/pages/GettingStarted.tsx index 8b2d4aebaad..d640a2d3786 100644 --- a/packages/webapp/src/pages/GettingStarted.tsx +++ b/packages/webapp/src/pages/GettingStarted.tsx @@ -216,7 +216,7 @@ nango.auth('${providerConfigKey}', '${connectionId}') model }; - const res = await fetch(`/sync/records?${new URLSearchParams(params).toString()}`, { + const res = await fetch(`/records?${new URLSearchParams(params).toString()}`, { method: 'GET', headers: { 'Authorization': `Bearer ${secretKey}`, diff --git a/packages/webapp/src/utils/language-snippets.tsx b/packages/webapp/src/utils/language-snippets.tsx index e8f8baf2791..4e991deafdd 100644 --- a/packages/webapp/src/utils/language-snippets.tsx +++ b/packages/webapp/src/utils/language-snippets.tsx @@ -14,7 +14,7 @@ console.log(issues); export const curlSnippet = (model: string, secretKey: string, connectionId: string, providerConfigKey: string) => { return ` curl --request GET \\ - --url https://api.nango.dev/sync/records?model=${model} \\ + --url https://api.nango.dev/records?model=${model} \\ --header 'Authorization: Bearer ${secretKey}' \\ --header 'Connection-Id: ${connectionId}' \\ --header 'Provider-Config-Key: ${providerConfigKey}' @@ -25,7 +25,7 @@ export const pythonSnippet = (model: string, secretKey: string, connectionId: st return` import requests -url = "https://api.nango.dev/sync/records" +url = "https://api.nango.dev/records" querystring = {"model":"${model}"} @@ -48,7 +48,7 @@ export const phpSnippet = (model: string, secretKey: string, connectionId: strin $curl = curl_init(); curl_setopt_array($curl, [ - CURLOPT_URL => "https://api.nango.dev/sync/records?model=${model}", + CURLOPT_URL => "https://api.nango.dev/records?model=${model}", CURLOPT_RETURNTRANSFER => true, CURLOPT_ENCODING => "", CURLOPT_MAXREDIRS => 10, @@ -86,7 +86,7 @@ import ( func main() { - url := "https://api.nango.dev/sync/records?model=${model}" + url := "https://api.nango.dev/records?model=${model}" req, _ := http.NewRequest("GET", url, nil) @@ -106,7 +106,7 @@ func main() { export const javaSnippet = (model: string, secretKey: string, connectionId: string, providerConfigKey: string) => { return` -HttpResponse response = Unirest.get("https://api.nango.dev/sync/records?model=${model}") +HttpResponse response = Unirest.get("https://api.nango.dev/records?model=${model}") .header("Authorization", "Bearer ${secretKey}") .header("Connection-Id", "${connectionId}") .header("Provider-Config-Key", "${providerConfigKey}") From dad5fdf25e56d8dd14cc2bf7a18460c2345c83ba Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 13:41:08 -0500 Subject: [PATCH 25/85] 0.36.61 --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4334d9149eb..56ab5c66e1f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15303,12 +15303,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.57", + "version": "0.36.61", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15409,7 +15409,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.58", + "version": "0.36.61", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15417,7 +15417,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15680,7 +15680,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.58", + "version": "0.36.61", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15697,7 +15697,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15722,11 +15722,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.58", + "version": "0.36.61", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15785,13 +15785,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.58", + "version": "0.36.61", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.58", + "@nangohq/node": "0.36.61", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16012,9 +16012,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.58", + "version": "0.36.61", "dependencies": { - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 4e3914f5852..5e2b3954c3b 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.57 + image: nangohq/nango-server:0.36.61 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.57 + image: nangohq/nango-worker:0.36.61 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index 590419edc28..d01d1509f65 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.57", + "version": "0.36.61", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 1dc477b8926..a671a7fbd64 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.58", + "version": "0.36.61", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index ace7bf913b2..2c0941fda4c 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 3a6c768a475..1af1fb49f7e 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.58", + "version": "0.36.61", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index 1aa073c4109..258b366e159 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index c1e364fd1d2..3695ff135e4 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.58", + "version": "0.36.61", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index d200a922fe3..dc6cab2bb9f 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.58", + "version": "0.36.61", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.58", + "@nangohq/node": "0.36.61", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index a3abc789f74..13539f94af2 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.58", + "version": "0.36.61", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.58", + "version": "0.36.61", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.58", + "@nangohq/frontend": "0.36.61", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.58", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.58.tgz", - "integrity": "sha512-fk+kC7ydjm4blSrwbac65iBD8Sj1Ntcuvs4l1W8Hf3e0+/w15WyPxWo6Yy4riL2oi1+U9b0NXYujWXEU+Edlqg==" + "version": "0.36.61", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.61.tgz", + "integrity": "sha512-ERlTfQ1B4bZ+2OHW7niax09Y9nxvpRi8W6sfjzq1+Tqma0MbpyMVIFqWhHl0VvUe0+FEp/Ww7QoapF+yGCvsrw==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.58", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.58.tgz", - "integrity": "sha512-fk+kC7ydjm4blSrwbac65iBD8Sj1Ntcuvs4l1W8Hf3e0+/w15WyPxWo6Yy4riL2oi1+U9b0NXYujWXEU+Edlqg==" + "version": "0.36.61", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.61.tgz", + "integrity": "sha512-ERlTfQ1B4bZ+2OHW7niax09Y9nxvpRi8W6sfjzq1+Tqma0MbpyMVIFqWhHl0VvUe0+FEp/Ww7QoapF+yGCvsrw==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index fd78f04aaf9..0b20ba51989 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.58", + "version": "0.36.61", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.58", + "@nangohq/frontend": "0.36.61", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 010b7b2b7d4..55b509c6858 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.58", + "version": "0.36.61", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.58", + "@nangohq/shared": "0.36.61", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From a4ad5a41580d3ab37f3517e9b914946315ec0c22 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 16:36:14 -0500 Subject: [PATCH 26/85] [nan-182] pass more information in flow delete call to perform the correct query (#1447) * [nan-182] pass more information in flow delete call to perform the correct query * [nan-182] complain about missing sync name --- .../server/lib/controllers/flow.controller.ts | 22 +++++++++++++++---- .../services/sync/config/config.service.ts | 18 --------------- .../shared/lib/services/sync/sync.service.ts | 20 +++++++++++++++++ packages/webapp/src/pages/Syncs.tsx | 3 ++- 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/packages/server/lib/controllers/flow.controller.ts b/packages/server/lib/controllers/flow.controller.ts index d505380c5c6..1f7f622873d 100644 --- a/packages/server/lib/controllers/flow.controller.ts +++ b/packages/server/lib/controllers/flow.controller.ts @@ -14,7 +14,7 @@ import { remoteFileService, getAllSyncsAndActions, getNangoConfigIdAndLocationFromId, - getSyncConfigConnections + getSyncsByConnectionIdsAndEnvironmentIdAndSyncName } from '@nangohq/shared'; class FlowController { @@ -207,16 +207,30 @@ class FlowController { const { environmentId } = response; const id = req.params['id']; + const connectionIds = req.query['connectionIds'] as string; + const syncName = req.query['sync_name'] as string; if (!id) { res.status(400).send('Missing id'); return; } - const syncsWithConnections = await getSyncConfigConnections(Number(id), environmentId); + if (!connectionIds) { + res.status(400).send('Missing connectionIds'); + return; + } + + if (!syncName) { + res.status(400).send('Missing sync_name'); + return; + } + + const connections = connectionIds.split(','); + + const syncs = await getSyncsByConnectionIdsAndEnvironmentIdAndSyncName(connections, environmentId, syncName); - for (const syncsWithConnection of syncsWithConnections) { - await syncOrchestrator.deleteSync(syncsWithConnection.id as string, environmentId); + for (const sync of syncs) { + await syncOrchestrator.deleteSync(sync.id as string, environmentId); } await syncOrchestrator.deleteConfig(Number(id), environmentId); diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts index 416da5d9027..0c68bf08f5a 100644 --- a/packages/shared/lib/services/sync/config/config.service.ts +++ b/packages/shared/lib/services/sync/config/config.service.ts @@ -510,24 +510,6 @@ export async function getSyncConfigsWithConnectionsByEnvironmentId(environment_i return result; } -export async function getSyncConfigConnections(id: number, environment_id: number): Promise<{ connection_id: string; id: string; name: string }[]> { - const result = await schema() - .select(`_nango_connections.connection_id`, '_nango_syncs.id', '_nango_syncs.name') - .from(TABLE) - .join('_nango_configs', `${TABLE}.nango_config_id`, '_nango_configs.id') - .join('_nango_connections', '_nango_connections.provider_config_key', '_nango_configs.unique_key') - .join('_nango_syncs', `_nango_syncs.name`, `${TABLE}.sync_name`) - .where({ - '_nango_configs.environment_id': environment_id, - active: true, - '_nango_configs.deleted': false, - [`${TABLE}.deleted`]: false, - [`${TABLE}.id`]: id - }); - - return result; -} - /** * Get Sync Configs By Provider Key * @desc grab all the sync configs by a provider key diff --git a/packages/shared/lib/services/sync/sync.service.ts b/packages/shared/lib/services/sync/sync.service.ts index ec76f717611..cef58a91185 100644 --- a/packages/shared/lib/services/sync/sync.service.ts +++ b/packages/shared/lib/services/sync/sync.service.ts @@ -450,6 +450,26 @@ export const findSyncByConnections = async (connectionIds: number[], sync_name: return []; }; +export const getSyncsByConnectionIdsAndEnvironmentIdAndSyncName = async (connectionIds: string[], environmentId: number, syncName: string): Promise => { + const results = await schema() + .select(`${TABLE}.id`) + .from(TABLE) + .join('_nango_connections', '_nango_connections.id', `${TABLE}.nango_connection_id`) + .whereIn('_nango_connections.connection_id', connectionIds) + .andWhere({ + name: syncName, + environment_id: environmentId, + [`${TABLE}.deleted`]: false, + [`_nango_connections.deleted`]: false + }); + + if (Array.isArray(results) && results.length > 0) { + return results; + } + + return []; +}; + export const getAndReconcileDifferences = async ( environmentId: number, syncs: IncomingFlowConfig[], diff --git a/packages/webapp/src/pages/Syncs.tsx b/packages/webapp/src/pages/Syncs.tsx index 53bcce05b65..7cc64bd983b 100644 --- a/packages/webapp/src/pages/Syncs.tsx +++ b/packages/webapp/src/pages/Syncs.tsx @@ -87,7 +87,8 @@ export default function Syncs() { } const onDeletePreBuilt = async () => { - const res = await fetch(`/api/v1/flow/${selectedFlowToDelete?.id}`, { + const connections = selectedFlowToDelete?.connections?.map((connection) => connection.connection_id).join(','); + const res = await fetch(`/api/v1/flow/${selectedFlowToDelete?.id}?sync_name=${selectedFlowToDelete?.sync_name}&connectionIds=${connections}`, { method: 'DELETE', headers: { 'Content-Type': 'application/json' From 8248e07990657614ba55b533fd93bb867e474bfa Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 16:43:47 -0500 Subject: [PATCH 27/85] 0.36.62 --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 56ab5c66e1f..83856f4ee8f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15303,12 +15303,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.61", + "version": "0.36.62", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15409,7 +15409,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.61", + "version": "0.36.62", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15417,7 +15417,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15680,7 +15680,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.61", + "version": "0.36.62", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15697,7 +15697,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15722,11 +15722,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.61", + "version": "0.36.62", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15785,13 +15785,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.61", + "version": "0.36.62", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.61", + "@nangohq/node": "0.36.62", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16012,9 +16012,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.61", + "version": "0.36.62", "dependencies": { - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 5e2b3954c3b..b7eee178c33 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.61 + image: nangohq/nango-server:0.36.62 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.61 + image: nangohq/nango-worker:0.36.62 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index d01d1509f65..2f03a1af5f5 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.61", + "version": "0.36.62", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index a671a7fbd64..7957242c7d8 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.61", + "version": "0.36.62", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 2c0941fda4c..6b994b2b736 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 1af1fb49f7e..48da3f2e666 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.61", + "version": "0.36.62", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index 258b366e159..ff1b41e44ac 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index 3695ff135e4..79ed12fa6ba 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.61", + "version": "0.36.62", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index dc6cab2bb9f..bcc0390aaa3 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.61", + "version": "0.36.62", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.61", + "@nangohq/node": "0.36.62", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index 13539f94af2..d4a9362b41e 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.61", + "version": "0.36.62", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.61", + "version": "0.36.62", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.61", + "@nangohq/frontend": "0.36.62", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.61", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.61.tgz", - "integrity": "sha512-ERlTfQ1B4bZ+2OHW7niax09Y9nxvpRi8W6sfjzq1+Tqma0MbpyMVIFqWhHl0VvUe0+FEp/Ww7QoapF+yGCvsrw==" + "version": "0.36.62", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.62.tgz", + "integrity": "sha512-Rgq7dIOSy0eHt+3vSKooDLkVIcy45uZxnhWO7L+pDtS0hFe7mFZAPtT1W4lS1nmcnyYJWJSC7AFmA8jJsXbBUg==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.61", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.61.tgz", - "integrity": "sha512-ERlTfQ1B4bZ+2OHW7niax09Y9nxvpRi8W6sfjzq1+Tqma0MbpyMVIFqWhHl0VvUe0+FEp/Ww7QoapF+yGCvsrw==" + "version": "0.36.62", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.62.tgz", + "integrity": "sha512-Rgq7dIOSy0eHt+3vSKooDLkVIcy45uZxnhWO7L+pDtS0hFe7mFZAPtT1W4lS1nmcnyYJWJSC7AFmA8jJsXbBUg==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 0b20ba51989..596cd99f1a3 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.61", + "version": "0.36.62", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.61", + "@nangohq/frontend": "0.36.62", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 55b509c6858..3836e85e871 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.61", + "version": "0.36.62", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.61", + "@nangohq/shared": "0.36.62", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From 0db905cd9b377b4c9e3d8c3863a4c99ee04f77df Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 20:15:49 -0500 Subject: [PATCH 28/85] [nan-182] create missing syncs even if some are active (#1448) --- .../shared/lib/services/sync/sync.service.ts | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/shared/lib/services/sync/sync.service.ts b/packages/shared/lib/services/sync/sync.service.ts index cef58a91185..ac4269e6888 100644 --- a/packages/shared/lib/services/sync/sync.service.ts +++ b/packages/shared/lib/services/sync/sync.service.ts @@ -520,8 +520,9 @@ export const getAndReconcileDifferences = async ( * When we come back here and performAction is true, the sync would have been created so exists will be true and we'll only create * the sync if there are connections */ + let syncsByConnection: Sync[] = []; if (exists && connections.length > 0) { - const syncsByConnection = await findSyncByConnections( + syncsByConnection = await findSyncByConnections( connections.map((connection) => connection.id as number), syncName ); @@ -548,6 +549,26 @@ export const getAndReconcileDifferences = async ( syncsToCreate.push({ connections, syncName, sync, providerConfigKey, environmentId }); } } + + // in some cases syncs are missing so let's also create them if missing + if (performAction && syncsByConnection.length !== 0 && syncsByConnection.length !== connections.length) { + const missingConnections = connections.filter((connection) => { + return !syncsByConnection.find((sync) => sync.nango_connection_id === connection.id); + }); + + if (missingConnections.length > 0) { + if (debug && activityLogId) { + await createActivityLogMessage({ + level: 'debug', + environment_id: environmentId, + activity_log_id: activityLogId as number, + timestamp: Date.now(), + content: `Creating sync ${syncName} for ${providerConfigKey} with ${missingConnections.length} connections` + }); + } + syncsToCreate.push({ connections: missingConnections, syncName, sync, providerConfigKey, environmentId }); + } + } } if (syncsToCreate.length > 0) { From 30b4758684d66df34682ffd0aff4cc037c5cb45f Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 19 Dec 2023 20:26:31 -0500 Subject: [PATCH 29/85] 0.36.63 --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 4 ++-- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 37 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index 83856f4ee8f..f988e5ebbd0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15303,12 +15303,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.62", + "version": "0.36.63", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15409,7 +15409,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.62", + "version": "0.36.63", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15417,7 +15417,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15680,7 +15680,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.62", + "version": "0.36.63", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15697,7 +15697,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" @@ -15722,11 +15722,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.62", + "version": "0.36.63", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15785,13 +15785,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.62", + "version": "0.36.63", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.62", + "@nangohq/node": "0.36.63", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16012,9 +16012,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.62", + "version": "0.36.63", "dependencies": { - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index b7eee178c33..69dfc9462fa 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.62 + image: nangohq/nango-server:0.36.63 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.62 + image: nangohq/nango-worker:0.36.63 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index 2f03a1af5f5..39e3f131344 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.62", + "version": "0.36.63", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 7957242c7d8..1c52df6b5fe 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.62", + "version": "0.36.63", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 6b994b2b736..82cceb8de89 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 48da3f2e666..83016396bdc 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.62", + "version": "0.36.63", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index ff1b41e44ac..e906812fe4c 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1" diff --git a/packages/server/package.json b/packages/server/package.json index 79ed12fa6ba..5cec813ede7 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.62", + "version": "0.36.63", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", diff --git a/packages/shared/package.json b/packages/shared/package.json index bcc0390aaa3..b463e1a52e0 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.62", + "version": "0.36.63", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.62", + "@nangohq/node": "0.36.63", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index d4a9362b41e..aab9888e5c5 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.62", + "version": "0.36.63", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.62", + "version": "0.36.63", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.62", + "@nangohq/frontend": "0.36.63", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.62", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.62.tgz", - "integrity": "sha512-Rgq7dIOSy0eHt+3vSKooDLkVIcy45uZxnhWO7L+pDtS0hFe7mFZAPtT1W4lS1nmcnyYJWJSC7AFmA8jJsXbBUg==" + "version": "0.36.63", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.63.tgz", + "integrity": "sha512-A2UfIbu63lqDMefsn+EzOZ01Z5rLX0v/G3vA3N2uRUnWq+u97K+NdIpe4y4ckiPz4GrcO6mIiobVuSkFpqo72w==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.62", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.62.tgz", - "integrity": "sha512-Rgq7dIOSy0eHt+3vSKooDLkVIcy45uZxnhWO7L+pDtS0hFe7mFZAPtT1W4lS1nmcnyYJWJSC7AFmA8jJsXbBUg==" + "version": "0.36.63", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.63.tgz", + "integrity": "sha512-A2UfIbu63lqDMefsn+EzOZ01Z5rLX0v/G3vA3N2uRUnWq+u97K+NdIpe4y4ckiPz4GrcO6mIiobVuSkFpqo72w==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 596cd99f1a3..16b02900585 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.62", + "version": "0.36.63", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.62", + "@nangohq/frontend": "0.36.63", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 3836e85e871..7536b2a758a 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.62", + "version": "0.36.63", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.62", + "@nangohq/shared": "0.36.63", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From f6db4c484ba7257414b096577065ba22147eacfc Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 08:26:01 +0100 Subject: [PATCH 30/85] jobs: fallback to default runner if account runner is not ready quickly (#1440) this commit is also removing the global temporal flag for jobs service as well as the runner-for-account flag --- packages/jobs/lib/app.ts | 25 ++++++++--------------- packages/jobs/lib/integration.service.ts | 15 ++++++-------- packages/jobs/lib/runner/local.runner.ts | 4 ++-- packages/jobs/lib/runner/render.runner.ts | 7 ++++--- packages/jobs/lib/runner/runner.ts | 8 +++++++- packages/jobs/lib/temporal.ts | 1 - 6 files changed, 27 insertions(+), 33 deletions(-) diff --git a/packages/jobs/lib/app.ts b/packages/jobs/lib/app.ts index eaf88f64d3f..f0811f13552 100644 --- a/packages/jobs/lib/app.ts +++ b/packages/jobs/lib/app.ts @@ -1,31 +1,22 @@ import { Temporal } from './temporal.js'; import { server } from './server.js'; -import { featureFlags } from '@nangohq/shared'; import './tracer.js'; try { const port = parseInt(process.env['NANGO_JOBS_PORT'] || '3005', 10); server.listen(port); console.log(`🚀 Jobs service ready at http://localhost:${port}`); - const temporalNs = process.env['TEMPORAL_NAMESPACE'] || 'default'; const temporal = new Temporal(temporalNs); + temporal.start(); - // TODO: remove flag check once jobs is fully enabled by default - let isTemporalStarted = false; - setInterval(async () => { - if (isTemporalStarted) { - return; - } - const isTemporalEnabled = await featureFlags.isEnabled('jobs-temporal', 'global', false); - if (isTemporalEnabled) { - isTemporalStarted = true; - temporal.start(); - } else { - temporal.stop(); - isTemporalStarted = false; - } - }, 5000); + // handle SIGTERM + process.on('SIGTERM', async () => { + temporal.stop(); + server.server.close(async () => { + process.exit(0); + }); + }); } catch (err) { console.error(`[JOBS]: ${err}`); process.exit(1); diff --git a/packages/jobs/lib/integration.service.ts b/packages/jobs/lib/integration.service.ts index 6a284785e22..2f288b22439 100644 --- a/packages/jobs/lib/integration.service.ts +++ b/packages/jobs/lib/integration.service.ts @@ -7,13 +7,12 @@ import { localFileService, remoteFileService, isCloud, - getEnv, + isProd, ServiceResponse, NangoError, - formatScriptError, - featureFlags + formatScriptError } from '@nangohq/shared'; -import { getRunner } from './runner/runner.js'; +import { getRunner, getRunnerId } from './runner/runner.js'; import tracer from './tracer.js'; class IntegrationService implements IntegrationServiceInterface { @@ -87,12 +86,10 @@ class IntegrationService implements IntegrationServiceInterface { } const accountId = nangoProps.accountId; - const isRunnerForAccountEnabled = await featureFlags.isEnabled('runner-for-account', `${accountId}`, false); - const runnerSuffix = isRunnerForAccountEnabled ? `${accountId}` : 'default'; - const runnerId = `${getEnv()}-runner-account-${runnerSuffix}`; - const runner = await getRunner(runnerId); + const runnerId = isProd() ? getRunnerId(`${accountId}`) : getRunnerId('default'); // a runner per account in prod only + const runner = await getRunner(runnerId).catch((_) => getRunner(getRunnerId('default'))); // fallback to default runner if account runner isn't ready yet - const runSpan = tracer.startSpan('runner.run', { childOf: span }).setTag('runnerId', runnerId); + const runSpan = tracer.startSpan('runner.run', { childOf: span }).setTag('runnerId', runner.id); try { // TODO: request sent to the runner for it to run the script is synchronous. // TODO: Make the request return immediately and have the runner ping the job service when it's done. diff --git a/packages/jobs/lib/runner/local.runner.ts b/packages/jobs/lib/runner/local.runner.ts index 01c9224a9bb..f806f084d20 100644 --- a/packages/jobs/lib/runner/local.runner.ts +++ b/packages/jobs/lib/runner/local.runner.ts @@ -3,7 +3,7 @@ import { execSync, spawn, ChildProcess } from 'child_process'; import { getRunnerClient } from '@nangohq/nango-runner'; export class LocalRunner implements Runner { - constructor(public readonly client: any, private readonly childProcess: ChildProcess) {} + constructor(public readonly id: string, public readonly client: any, private readonly childProcess: ChildProcess) {} async stop(): Promise { this.childProcess.kill(); @@ -47,7 +47,7 @@ export class LocalRunner implements Runner { } const client = getRunnerClient(`http://localhost:${port}`); - return new LocalRunner(client, childProcess); + return new LocalRunner(runnerId, client, childProcess); } catch (err) { throw new Error(`Unable to get runner ${runnerId}: ${err}`); } diff --git a/packages/jobs/lib/runner/render.runner.ts b/packages/jobs/lib/runner/render.runner.ts index 042bfa4df36..9aa9ca1dfe6 100644 --- a/packages/jobs/lib/runner/render.runner.ts +++ b/packages/jobs/lib/runner/render.runner.ts @@ -7,7 +7,7 @@ const render = api('@render-api/v1.0#aiie8wizhlp1is9bu'); render.auth(process.env['RENDER_API_KEY']); export class RenderRunner implements Runner { - constructor(public readonly client: any, private readonly serviceId: string) {} + constructor(public readonly id: string, public readonly client: any, private readonly serviceId: string) {} async stop(): Promise { render.suspendService({ serviceId: this.serviceId }); @@ -40,7 +40,8 @@ export class RenderRunner implements Runner { { key: 'NANGO_DB_PASSWORD', value: process.env['NANGO_DB_PASSWORD'] }, { key: 'NANGO_DB_PORT', value: process.env['NANGO_DB_PORT'] }, { key: 'NANGO_DB_SSL', value: process.env['NANGO_DB_SSL'] }, - { key: 'NANGO_ENCRYPTION_KEY', value: process.env['NANGO_ENCRYPTION_KEY'] } + { key: 'NANGO_ENCRYPTION_KEY', value: process.env['NANGO_ENCRYPTION_KEY'] }, + { key: 'NODE_OPTIONS', value: '--max-old-space-size=384' } ] }); svc = res.data.service; @@ -54,7 +55,7 @@ export class RenderRunner implements Runner { console.log(res); } const client = getRunnerClient(`http://${runnerId}`); - return new RenderRunner(client, svc.id); + return new RenderRunner(runnerId, client, svc.id); } catch (err) { throw new Error(`Unable to get runner ${runnerId}: ${err}`); } diff --git a/packages/jobs/lib/runner/runner.ts b/packages/jobs/lib/runner/runner.ts index 1f9ccb06dfb..82b98de5de9 100644 --- a/packages/jobs/lib/runner/runner.ts +++ b/packages/jobs/lib/runner/runner.ts @@ -1,12 +1,17 @@ import { LocalRunner } from './local.runner.js'; import { RenderRunner } from './render.runner.js'; +import { getEnv } from '@nangohq/shared'; + +export function getRunnerId(suffix: string): string { + return `${getEnv()}-runner-account-${suffix}`; +} export async function getRunner(runnerId: string): Promise { const isRender = process.env['IS_RENDER'] === 'true'; const runner = isRender ? await RenderRunner.get(runnerId) : await LocalRunner.get(runnerId); // Wait for runner to start and be healthy - const timeoutMs = isRender ? 90000 : 10000; + const timeoutMs = 5000; let healthCheck = false; let startTime = Date.now(); while (!healthCheck && Date.now() - startTime < timeoutMs) { @@ -24,6 +29,7 @@ export async function getRunner(runnerId: string): Promise { } export interface Runner { + id: string; client: any; stop(): Promise; } diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts index 50ff9ec6d6c..d7a06185d53 100644 --- a/packages/jobs/lib/temporal.ts +++ b/packages/jobs/lib/temporal.ts @@ -66,7 +66,6 @@ export class Temporal { stop() { if (this.workers) { - console.log('Stopping Temporal worker'); this.workers.forEach((worker) => worker.shutdown()); } } From 3c247c38f6a5c791a345df58dc9f932a04f82e39 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:03:45 +0100 Subject: [PATCH 31/85] pin dd-trace to 4.20.0 (#1450) it looks like v4.21.0 have introduced a memory leak. See https://github.com/DataDog/dd-trace-js/issues/3887 We are seeing some crash in nango-server due to `RangeError: Maximum call stack size exceeded` error. Pinning dd-trace to 4.20.0 which doesn't seem to be affected by the memory leak until the problem is fixed --- package-lock.json | 306 +++++++++++++++++++++++------------ packages/jobs/package.json | 4 +- packages/server/package.json | 4 +- 3 files changed, 206 insertions(+), 108 deletions(-) diff --git a/package-lock.json b/package-lock.json index f988e5ebbd0..2870d88111c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2820,18 +2820,6 @@ "node": ">=12.0.0" } }, - "node_modules/@datadog/native-appsec": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-5.0.0.tgz", - "integrity": "sha512-Ks8a4L49N40w+TJjj2e9ncGssUIEjo4wnmUFjPBRvlLGuVj1VJLxCx7ztpd8eTycM5QQlzggCDOP6CMEVmeZbA==", - "hasInstallScript": true, - "dependencies": { - "node-gyp-build": "^3.9.0" - }, - "engines": { - "node": ">=12" - } - }, "node_modules/@datadog/native-iast-rewriter": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.2.1.tgz", @@ -2884,22 +2872,6 @@ "node": ">=12" } }, - "node_modules/@datadog/pprof": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-4.1.0.tgz", - "integrity": "sha512-g7EWI185nwSuFwlmnAGDPxbPsqe+ipOoDB2oP841WMNRaJBPRdg5J90c+6ucmyltuC9VpTrmzzqcachkOTzZEQ==", - "hasInstallScript": true, - "dependencies": { - "delay": "^5.0.0", - "node-gyp-build": "<4.0", - "p-limit": "^3.1.0", - "pprof-format": "^2.0.7", - "source-map": "^0.7.4" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/@datadog/sketches-js": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/@datadog/sketches-js/-/sketches-js-2.1.0.tgz", @@ -7857,75 +7829,6 @@ "node": ">=12.17" } }, - "node_modules/dd-trace": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-4.21.0.tgz", - "integrity": "sha512-nuFHfDJNy039Qns1sbJdIbXeP1ZmUioctwzXfw60k4Dif8dC3RmrHebvW48EbYCKrJj3Awri/vmwcBNf6xNpHw==", - "hasInstallScript": true, - "dependencies": { - "@datadog/native-appsec": "5.0.0", - "@datadog/native-iast-rewriter": "2.2.1", - "@datadog/native-iast-taint-tracking": "1.6.4", - "@datadog/native-metrics": "^2.0.0", - "@datadog/pprof": "4.1.0", - "@datadog/sketches-js": "^2.1.0", - "@opentelemetry/api": "^1.0.0", - "@opentelemetry/core": "^1.14.0", - "crypto-randomuuid": "^1.0.0", - "dc-polyfill": "^0.1.2", - "ignore": "^5.2.4", - "import-in-the-middle": "^1.4.2", - "int64-buffer": "^0.1.9", - "ipaddr.js": "^2.1.0", - "istanbul-lib-coverage": "3.2.0", - "jest-docblock": "^29.7.0", - "koalas": "^1.0.2", - "limiter": "^1.1.4", - "lodash.kebabcase": "^4.1.1", - "lodash.pick": "^4.4.0", - "lodash.sortby": "^4.7.0", - "lodash.uniq": "^4.5.0", - "lru-cache": "^7.14.0", - "methods": "^1.1.2", - "module-details-from-path": "^1.0.3", - "msgpack-lite": "^0.1.26", - "node-abort-controller": "^3.1.1", - "opentracing": ">=0.12.1", - "path-to-regexp": "^0.1.2", - "pprof-format": "^2.0.7", - "protobufjs": "^7.2.5", - "retry": "^0.13.1", - "semver": "^7.5.4", - "tlhunter-sorted-set": "^0.1.0" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/dd-trace/node_modules/ipaddr.js": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", - "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", - "engines": { - "node": ">= 10" - } - }, - "node_modules/dd-trace/node_modules/lru-cache": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", - "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", - "engines": { - "node": ">=12" - } - }, - "node_modules/dd-trace/node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "engines": { - "node": ">= 4" - } - }, "node_modules/debug": { "version": "4.3.4", "license": "MIT", @@ -13687,11 +13590,6 @@ "node": ">=14.0.0" } }, - "node_modules/tlhunter-sorted-set": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/tlhunter-sorted-set/-/tlhunter-sorted-set-0.1.0.tgz", - "integrity": "sha512-eGYW4bjf1DtrHzUYxYfAcSytpOkA44zsr7G2n3PV7yOUR23vmkGe3LL4R+1jL9OsXtbsFOwe8XtbCrabeaEFnw==" - }, "node_modules/tmp": { "version": "0.0.33", "license": "MIT", @@ -15428,7 +15326,7 @@ "@trpc/client": "^10.44.1", "@trpc/server": "^10.44.1", "@types/fs-extra": "^11.0.1", - "dd-trace": "^4.21.0", + "dd-trace": "4.20.0", "dotenv": "^16.0.3", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", @@ -15450,6 +15348,34 @@ "typescript": "^5.3.2" } }, + "packages/jobs/node_modules/@datadog/native-appsec": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-4.0.0.tgz", + "integrity": "sha512-myTguXJ3VQHS2E1ylNsSF1avNpDmq5t+K4Q47wdzeakGc3sDIDDyEbvuFTujl9c9wBIkup94O1mZj5DR37ajzA==", + "hasInstallScript": true, + "dependencies": { + "node-gyp-build": "^3.9.0" + }, + "engines": { + "node": ">=12" + } + }, + "packages/jobs/node_modules/@datadog/pprof": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-4.0.1.tgz", + "integrity": "sha512-TavqyiyQZOaUM9eQB07r8+K/T1CqKyOdsUGxpN79+BF+eOQBpTj/Cte6KdlhcUSKL3h5hSjC+vlgA7uW2qtVhA==", + "hasInstallScript": true, + "dependencies": { + "delay": "^5.0.0", + "node-gyp-build": "<4.0", + "p-limit": "^3.1.0", + "pprof-format": "^2.0.7", + "source-map": "^0.7.4" + }, + "engines": { + "node": ">=14" + } + }, "packages/jobs/node_modules/@eslint/eslintrc": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", @@ -15518,6 +15444,58 @@ "sprintf-js": "~1.0.2" } }, + "packages/jobs/node_modules/dd-trace": { + "version": "4.20.0", + "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-4.20.0.tgz", + "integrity": "sha512-y7IDLSSt6nww6zMdw/I8oLdfgOQADIOkERCNdfSzlBrXHz5CHimEOFfsHN87ag0mn6vusr06aoi+CQRZSNJz2g==", + "hasInstallScript": true, + "dependencies": { + "@datadog/native-appsec": "4.0.0", + "@datadog/native-iast-rewriter": "2.2.1", + "@datadog/native-iast-taint-tracking": "1.6.4", + "@datadog/native-metrics": "^2.0.0", + "@datadog/pprof": "4.0.1", + "@datadog/sketches-js": "^2.1.0", + "@opentelemetry/api": "^1.0.0", + "@opentelemetry/core": "^1.14.0", + "crypto-randomuuid": "^1.0.0", + "dc-polyfill": "^0.1.2", + "ignore": "^5.2.4", + "import-in-the-middle": "^1.4.2", + "int64-buffer": "^0.1.9", + "ipaddr.js": "^2.1.0", + "istanbul-lib-coverage": "3.2.0", + "jest-docblock": "^29.7.0", + "koalas": "^1.0.2", + "limiter": "^1.1.4", + "lodash.kebabcase": "^4.1.1", + "lodash.pick": "^4.4.0", + "lodash.sortby": "^4.7.0", + "lodash.uniq": "^4.5.0", + "lru-cache": "^7.14.0", + "methods": "^1.1.2", + "module-details-from-path": "^1.0.3", + "msgpack-lite": "^0.1.26", + "node-abort-controller": "^3.1.1", + "opentracing": ">=0.12.1", + "path-to-regexp": "^0.1.2", + "pprof-format": "^2.0.7", + "protobufjs": "^7.2.4", + "retry": "^0.13.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "packages/jobs/node_modules/dd-trace/node_modules/ignore": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.0.tgz", + "integrity": "sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==", + "engines": { + "node": ">= 4" + } + }, "packages/jobs/node_modules/eslint": { "version": "7.32.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", @@ -15665,6 +15643,30 @@ "node": ">= 4" } }, + "packages/jobs/node_modules/ipaddr.js": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", + "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", + "engines": { + "node": ">= 10" + } + }, + "packages/jobs/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "packages/jobs/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, "packages/jobs/node_modules/typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -15733,7 +15735,7 @@ "connect-session-knex": "^3.0.1", "cookie-parser": "^1.4.6", "cors": "^2.8.5", - "dd-trace": "^4.19.0", + "dd-trace": "4.20.0", "dotenv": "^16.0.3", "exponential-backoff": "^3.1.1", "express": "^4.18.2", @@ -15783,6 +15785,102 @@ "npm": ">=6.14.11" } }, + "packages/server/node_modules/@datadog/native-appsec": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@datadog/native-appsec/-/native-appsec-4.0.0.tgz", + "integrity": "sha512-myTguXJ3VQHS2E1ylNsSF1avNpDmq5t+K4Q47wdzeakGc3sDIDDyEbvuFTujl9c9wBIkup94O1mZj5DR37ajzA==", + "hasInstallScript": true, + "dependencies": { + "node-gyp-build": "^3.9.0" + }, + "engines": { + "node": ">=12" + } + }, + "packages/server/node_modules/@datadog/pprof": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@datadog/pprof/-/pprof-4.0.1.tgz", + "integrity": "sha512-TavqyiyQZOaUM9eQB07r8+K/T1CqKyOdsUGxpN79+BF+eOQBpTj/Cte6KdlhcUSKL3h5hSjC+vlgA7uW2qtVhA==", + "hasInstallScript": true, + "dependencies": { + "delay": "^5.0.0", + "node-gyp-build": "<4.0", + "p-limit": "^3.1.0", + "pprof-format": "^2.0.7", + "source-map": "^0.7.4" + }, + "engines": { + "node": ">=14" + } + }, + "packages/server/node_modules/dd-trace": { + "version": "4.20.0", + "resolved": "https://registry.npmjs.org/dd-trace/-/dd-trace-4.20.0.tgz", + "integrity": "sha512-y7IDLSSt6nww6zMdw/I8oLdfgOQADIOkERCNdfSzlBrXHz5CHimEOFfsHN87ag0mn6vusr06aoi+CQRZSNJz2g==", + "hasInstallScript": true, + "dependencies": { + "@datadog/native-appsec": "4.0.0", + "@datadog/native-iast-rewriter": "2.2.1", + "@datadog/native-iast-taint-tracking": "1.6.4", + "@datadog/native-metrics": "^2.0.0", + "@datadog/pprof": "4.0.1", + "@datadog/sketches-js": "^2.1.0", + "@opentelemetry/api": "^1.0.0", + "@opentelemetry/core": "^1.14.0", + "crypto-randomuuid": "^1.0.0", + "dc-polyfill": "^0.1.2", + "ignore": "^5.2.4", + "import-in-the-middle": "^1.4.2", + "int64-buffer": "^0.1.9", + "ipaddr.js": "^2.1.0", + "istanbul-lib-coverage": "3.2.0", + "jest-docblock": "^29.7.0", + "koalas": "^1.0.2", + "limiter": "^1.1.4", + "lodash.kebabcase": "^4.1.1", + "lodash.pick": "^4.4.0", + "lodash.sortby": "^4.7.0", + "lodash.uniq": "^4.5.0", + "lru-cache": "^7.14.0", + "methods": "^1.1.2", + "module-details-from-path": "^1.0.3", + "msgpack-lite": "^0.1.26", + "node-abort-controller": "^3.1.1", + "opentracing": ">=0.12.1", + "path-to-regexp": "^0.1.2", + "pprof-format": "^2.0.7", + "protobufjs": "^7.2.4", + "retry": "^0.13.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=16" + } + }, + "packages/server/node_modules/ipaddr.js": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz", + "integrity": "sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==", + "engines": { + "node": ">= 10" + } + }, + "packages/server/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "packages/server/node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "engines": { + "node": ">= 4" + } + }, "packages/shared": { "name": "@nangohq/shared", "version": "0.36.63", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 82cceb8de89..70f9644ecc5 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -27,7 +27,7 @@ "@trpc/client": "^10.44.1", "@trpc/server": "^10.44.1", "@types/fs-extra": "^11.0.1", - "dd-trace": "^4.21.0", + "dd-trace": "4.20.0", "dotenv": "^16.0.3", "fs-extra": "^11.1.1", "js-yaml": "^4.1.0", @@ -48,4 +48,4 @@ "nodemon": "^3.0.1", "typescript": "^5.3.2" } -} +} \ No newline at end of file diff --git a/packages/server/package.json b/packages/server/package.json index 5cec813ede7..61a14bc3c56 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -29,7 +29,7 @@ "connect-session-knex": "^3.0.1", "cookie-parser": "^1.4.6", "cors": "^2.8.5", - "dd-trace": "^4.19.0", + "dd-trace": "4.20.0", "dotenv": "^16.0.3", "exponential-backoff": "^3.1.1", "express": "^4.18.2", @@ -74,4 +74,4 @@ "nodemon": "^3.0.1", "typescript": "^4.7.4" } -} +} \ No newline at end of file From 563bd7f79895bf09c433eed72b7d693ae35e04ce Mon Sep 17 00:00:00 2001 From: Hassan_Wari <85742599+hassan254-prog@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:25:09 +0300 Subject: [PATCH 32/85] Add integration templates for zoho-crm (#1438) * Add integration templates for zoho-crm * Made changes to zoho-crm-accounts * Removed trailing spaces * Added a new line at end of file * Made changes to contacts and deals sync scripts --------- Co-authored-by: Bastien Beurier --- docs-v2/integration-templates/overview.mdx | 4 + docs-v2/integration-templates/zoho-crm.mdx | 21 ++ docs-v2/mint.json | 3 +- integration-templates/zoho-crm/nango.yaml | 245 ++++++++++++++++++ .../zoho-crm/zoho-crm-accounts.ts | 90 +++++++ .../zoho-crm/zoho-crm-contacts.ts | 100 +++++++ .../zoho-crm/zoho-crm-deals.ts | 80 ++++++ packages/shared/providers.yaml | 5 + 8 files changed, 547 insertions(+), 1 deletion(-) create mode 100644 docs-v2/integration-templates/zoho-crm.mdx create mode 100644 integration-templates/zoho-crm/nango.yaml create mode 100644 integration-templates/zoho-crm/zoho-crm-accounts.ts create mode 100644 integration-templates/zoho-crm/zoho-crm-contacts.ts create mode 100644 integration-templates/zoho-crm/zoho-crm-deals.ts diff --git a/docs-v2/integration-templates/overview.mdx b/docs-v2/integration-templates/overview.mdx index 2fde6ff023b..f2924747fb7 100644 --- a/docs-v2/integration-templates/overview.mdx +++ b/docs-v2/integration-templates/overview.mdx @@ -92,6 +92,10 @@ As Nango and its community expand, we're looking forward to offering hundreds of Sync Zendesk tickets and articles + + + Sync Zoho CRM accounts, contacts and deals/opportunities + ## Need Help with a Template? diff --git a/docs-v2/integration-templates/zoho-crm.mdx b/docs-v2/integration-templates/zoho-crm.mdx new file mode 100644 index 00000000000..b2a24b7e86f --- /dev/null +++ b/docs-v2/integration-templates/zoho-crm.mdx @@ -0,0 +1,21 @@ +--- +title: 'Zoho CRM API Integration Template' +sidebarTitle: 'Zoho CRM' +--- + +## Get started with the Zoho CRM template + + + Learn how to use integration templates in Nango + + + + Get the latest version of the Zoho CRM integration template from GitHub + + +## Need help with the template? +Please reach out on the [Slack community](https://nango.dev/slack), we are very active there and happy to help! \ No newline at end of file diff --git a/docs-v2/mint.json b/docs-v2/mint.json index 6a113343d5e..73259d2296d 100644 --- a/docs-v2/mint.json +++ b/docs-v2/mint.json @@ -99,7 +99,8 @@ "integration-templates/notion", "integration-templates/salesforce", "integration-templates/slack", - "integration-templates/zendesk" + "integration-templates/zendesk", + "integration-templates/zoho-crm" ] } ] diff --git a/integration-templates/zoho-crm/nango.yaml b/integration-templates/zoho-crm/nango.yaml new file mode 100644 index 00000000000..03463a68938 --- /dev/null +++ b/integration-templates/zoho-crm/nango.yaml @@ -0,0 +1,245 @@ +integrations: + zoho-crm: + zoho-crm-accounts: + runs: every half hour + auto_start: false + returns: + - ZohoCRMAccount + description: | + Fetches accounts from zoho crm. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): ZohoCRM.modules.accounts.READ or ZohoCRM.modules.ALL + zoho-crm-contacts: + runs: every half hour + auto_start: false + returns: + - ZohoCRMContact + description: | + Fetches contacts from zoho crm. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): ZohoCRM.modules.contacts.READ or ZohoCRM.modules.ALL + zoho-crm-deals: + runs: every half hour + auto_start: false + returns: + - ZohoCRMDeal + description: | + Fetches deals/opportunities from zoho crm. + Details: full sync, doesn't track deletes, metadata is not required. + Scope(s): ZohoCRM.modules.deals.READ or ZohoCRM.modules.ALL +models: + ZohoCRMAccount: + Owner: + name: string + id: string + email: string + $currency_symbol: string + $field_states: string + Account_Type: string + SIC_Code: string + Last_Activity_Time: date + Industry: string + Account_Site: string + $state: string + $process_flow: boolean + Billing_Country: string + $locked_for_me: boolean + id: string + $approved: boolean + $approval: + delegate: boolean + approve: boolean + reject: boolean + resubmit: boolean + Billing_Street: string + Created_Time: date + $editable: boolean + Billing_Code: string + Shipping_City: string + Shipping_Country: string + Shipping_Code: string + Billing_City: string + Created_By: + name: string + id: string + email: string + $zia_owner_assignment: string + Annual_Revenue: integer + Shipping_Street: string + Ownership: string + Description: string + Rating: integer + Shipping_State: string + $review_process: + approve: boolean + reject: boolean + resubmit: boolean + Website: string + Employees: integer + Record_Image: string + Modified_By: + name: string + id: string + email: string + $review: string + Phone: string + Account_Name: string + Account_Number: string + Ticker_Symbol: string + Modified_Time: date + $orchestration: boolean + Parent_Account: + name: string + id: string + $in_merge: boolean + Locked__s: boolean + Billing_State: string + Tag: [] + Fax: string + $approval_state: string + ZohoCRMContact: + Owner: + name: string + id: string + email: string + Email: string + $currency_symbol: string + $field_states: string + Other_Phone: string + Mailing_State: string + Other_State: string + Other_Country: string + Last_Activity_Time: date + Department: string + $state: string + Unsubscribed_Mode: string + $process_flow: boolean + Assistant: string + Mailing_Country: string + $locked_for_me: string + id: string + $approved: boolean + Reporting_To: + name: string + id: string + $approval: + delegate: boolean + approve: boolean + reject: boolean + resubmit: boolean + Other_City: string + Created_Time: date + $editable: boolean + Home_Phone: string + Created_By: + name: string + id: string + email: string + $zia_owner_assignment: string + Secondary_Email: string + Description: string + Vendor_Name: + name: string + id: string + Mailing_Zip: string + $review_process: + approve: boolean + reject: boolean + resubmit: boolean + Twitter: string + Other_Zip: string + Mailing_Street: string + Salutation: string + First_Name: string + Full_Name: string + Asst_Phone: string + Record_Image: string + Modified_By: + name: string + id: string + email: string + $review: boolean + Skype_ID: string + Phone: string + Account_Name: + name: string + id: string + Email_Opt_Out: boolean + Modified_Time: date + Date_of_Birth: date + Mailing_City: string + Unsubscribed_Time: date + Title: string + Other_Street: string + Mobile: string + $orchestration: boolean + Last_Name: string + $in_merge: boolean + Locked__s: boolean + Lead_Source: string + Tag: [] + Fax: string + $approval_state: string + ZohoCRMDeal: + Owner: + name: string + id: string + email: string + Description: string + $currency_symbol: string + Campaign_Source: + name: string + id: string + $field_states: string + $review_process: + approve: boolean + reject: boolean + resubmit: boolean + Closing_Date: date + Reason_For_Loss__s: string + Last_Activity_Time: date + Modified_By: + name: string + id: string + email: string + $review: string + Lead_Conversion_Time: date + $state: string + $process_flow: boolean + Deal_Name: string + Expected_Revenue: integer + Overall_Sales_Duration: integer + Stage: string + $locked_for_me: boolean + Account_Name: + name: string + id: string + id: string + $approved: boolean + $approval: + delegate: boolean + approve: boolean + reject: boolean + resubmit: boolean + Modified_Time: date + Created_Time: date + Amount: integer + Next_Step: string + Probability: integer + $editable: boolean + $orchestration: boolean + Contact_Name: + name: string + id: string + Sales_Cycle_Duration: integer + Type: string + $in_merge: boolean + Locked__s: boolean + Lead_Source: string + Created_By: + name: string + id: string + email: string + Tag: [] + $zia_owner_assignment: string + $approval_state: string diff --git a/integration-templates/zoho-crm/zoho-crm-accounts.ts b/integration-templates/zoho-crm/zoho-crm-accounts.ts new file mode 100644 index 00000000000..295a95b2663 --- /dev/null +++ b/integration-templates/zoho-crm/zoho-crm-accounts.ts @@ -0,0 +1,90 @@ +import type { ZohoCRMAccount, NangoSync } from './models'; + +export default async function fetchData(nango: NangoSync) { + let totalRecords = 0; + const fields = ''; // Define your fields to retrieve specific field values + + try { + const endpoint = '/crm/v2/Accounts'; + const config = { + headers: { + 'If-Modified-Since': nango.lastSyncDate?.toUTCString() || '' + }, + paginate: { + limit: 100 + }, + ...(fields ? { params: { fields } } : {}) + }; + for await (const account of nango.paginate({ ...config, endpoint })) { + const mappedAccounts: ZohoCRMAccount[] = account.map(mapAccounts) || []; + // Save Accounts + const batchSize: number = mappedAccounts.length; + totalRecords += batchSize; + + await nango.log(`Saving batch of ${batchSize} accounts (total accounts: ${totalRecords})`); + await nango.batchSave(mappedAccounts, 'ZohoCRMAccount'); + } + } catch (error: any) { + if (error.status = 304) { + await nango.log('No Accounts found.'); + } + else{ + throw new Error(`Error in fetchData: ${error.message}`); + } + } +} + +function mapAccounts(account: any): ZohoCRMAccount { + return { + Owner: account.Owner, + $currency_symbol: account.$currency_symbol, + $field_states: account.$field_states, + Account_Type: account.Account_Type, + SIC_Code: account.SIC_Code, + Last_Activity_Time: account.Last_Activity_Time, + Industry: account.Industry, + Account_Site: account.Account_Site, + $state: account.$state, + $process_flow: account.$process_flow, + Billing_Country: account.Billing_Country, + $locked_for_me: account.$locked_for_me, + id: account.id as string, + $approved: account.$approved, + $approval: account.$approval, + Billing_Street: account.Billing_Street, + Created_Time: account.Created_Time, + $editable: account.$editable, + Billing_Code: account.Billing_Code, + Shipping_City: account.Shipping_City, + Shipping_Country: account.Shipping_Country, + Shipping_Code: account.Shipping_Code, + Billing_City: account.Billing_City, + Created_By: account.Created_By, + $zia_owner_assignment: account.$zia_owner_assignment, + Annual_Revenue: account.Annual_Revenue, + Shipping_Street: account.Shipping_Street, + Ownership: account.Ownership, + Description: account.Description, + Rating: account.Rating, + Shipping_State: account.Shipping_State, + $review_process: account.$review_process, + Website: account.Website, + Employees: account.Employees, + Record_Image: account.Record_Image, + Modified_By: account.Modified_By, + $review: account.$review, + Phone: account.Phone, + Account_Name: account.Account_Name, + Account_Number: account.Account_Number, + Ticker_Symbol: account.Ticker_Symbol, + Modified_Time: account.Modified_Time, + $orchestration: account.$orchestration, + Parent_Account: account.Parent_Account, + $in_merge: account.$in_merge, + Locked__s: account.Locked__s, + Billing_State: account.Billing_State, + Tag: account.Tag, + Fax: account.Fax, + $approval_state: account.$approval_state + }; +} \ No newline at end of file diff --git a/integration-templates/zoho-crm/zoho-crm-contacts.ts b/integration-templates/zoho-crm/zoho-crm-contacts.ts new file mode 100644 index 00000000000..5ddc0c447a4 --- /dev/null +++ b/integration-templates/zoho-crm/zoho-crm-contacts.ts @@ -0,0 +1,100 @@ +import type { ZohoCRMContact, NangoSync } from './models'; + +export default async function fetchData(nango: NangoSync) { + let totalRecords = 0; + const fields = ''; // Define your fields to retrieve specific field values + + try { + const endpoint = '/crm/v2/Contacts'; + const config = { + headers: { + 'If-Modified-Since': nango.lastSyncDate?.toUTCString() || '' + }, + paginate: { + limit: 100 + }, + ...(fields ? { params: { fields } } : {}) + }; + for await (const contact of nango.paginate({ ...config, endpoint })) { + const mappedContacts: ZohoCRMContact[] = contact.map(mapContacts) || []; + // Save Contacts + const batchSize: number = mappedContacts.length; + totalRecords += batchSize; + + await nango.log(`Saving batch of ${batchSize} contacts (total contacts: ${totalRecords})`); + await nango.batchSave(mappedContacts, 'ZohoCRMContact'); + } + } catch (error: any) { + if (error.status = 304) { + await nango.log('No Contacts found.'); + } + else{ + throw new Error(`Error in fetchData: ${error.message}`); + } + } +} + +function mapContacts(contact: any): ZohoCRMContact { + return { + Owner: contact.Owner, + Email: contact.Email, + $currency_symbol: contact.$currency_symbol, + $field_states: contact.$field_states, + Other_Phone: contact.Other_Phone, + Mailing_State: contact.Mailing_State, + Other_State: contact.Other_State, + Other_Country: contact.Other_Country, + Last_Activity_Time: contact.Last_Activity_Time, + Department: contact.Department, + $state: contact.$state, + Unsubscribed_Mode: contact.Unsubscribed_Mode, + $process_flow: contact.$process_flow, + Assistant: contact.Assistant, + Mailing_Country: contact.Mailing_Country, + $locked_for_me: contact.locked_for_me, + id: contact.id as string, + $approved: contact.$approved, + Reporting_To: contact.Reporting_To, + $approval: contact.$approval, + Other_City: contact.Other_City, + Created_Time: contact.Created_Time, + $editable: contact.$editable, + Home_Phone: contact.Home_Phone, + Created_By: contact.Created_By, + $zia_owner_assignment: contact.$zia_owner_assignment, + Secondary_Email: contact.Secondary_Email, + Description: contact.Description, + Vendor_Name: contact.Vendor_Name, + Mailing_Zip: contact.Mailing_Zip, + $review_process: contact.$review_process, + Twitter: contact.Twitter, + Other_Zip: contact.Other_Zip, + Mailing_Street: contact.Mailing_Street, + Salutation: contact.Salutation, + First_Name: contact.First_Name, + Full_Name: contact.Full_Name, + Asst_Phone: contact.Asst_Phone, + Record_Image: contact.Record_Image, + Modified_By: contact.Modified_By, + $review: contact.$review, + Skype_ID: contact.Skype_ID, + Phone: contact.Phone, + Account_Name: contact.Account_Name, + Email_Opt_Out: contact.Email_Opt_Out, + Modified_Time: contact.Modified_Time, + Date_of_Birth: contact.Date_of_Birth, + Mailing_City: contact.Mailing_City, + Unsubscribed_Time: contact.Unsubscribed_Time, + Title: contact.Title, + Other_Street: contact.Other_Street, + Mobile: contact.Mobile, + $orchestration: contact.$orchestration, + Last_Name: contact.Last_Name, + $in_merge: contact.$in_merge, + Locked__s: contact.Locked__s, + Lead_Source: contact.Lead_Source, + Tag: contact.Tag, + Fax: contact.Fax, + $approval_state: contact.$approval_state + }; +} \ No newline at end of file diff --git a/integration-templates/zoho-crm/zoho-crm-deals.ts b/integration-templates/zoho-crm/zoho-crm-deals.ts new file mode 100644 index 00000000000..6ccb7a97db3 --- /dev/null +++ b/integration-templates/zoho-crm/zoho-crm-deals.ts @@ -0,0 +1,80 @@ +import type { ZohoCRMDeal, NangoSync } from './models'; + +export default async function fetchData(nango: NangoSync) { + let totalRecords = 0; + const fields = ''; // Define your fields to retrieve specific field values + + try { + const endpoint = '/crm/v2/Deals'; + const config = { + headers: { + 'If-Modified-Since': nango.lastSyncDate?.toUTCString() || '' + }, + paginate: { + limit: 100 + }, + ...(fields ? { params: { fields } } : {}) + }; + for await (const deal of nango.paginate({ ...config, endpoint })) { + const mappedDeals: ZohoCRMDeal[] = deal.map(mapDeals) || []; + // Save Deals + const batchSize: number = mappedDeals.length; + totalRecords += batchSize; + + await nango.log(`Saving batch of ${batchSize} deals (total deals: ${totalRecords})`); + await nango.batchSave(mappedDeals, 'ZohoCRMDeal'); + } + } catch (error: any) { + if (error.status = 304) { + await nango.log('No Deals found.'); + } + else{ + throw new Error(`Error in fetchData: ${error.message}`); + } + } +} + +function mapDeals(deal: any): ZohoCRMDeal { + return { + Owner: deal.Owner, + Description: deal.Description, + $currency_symbol: deal.$currency_symbol, + Campaign_Source: deal.Campaign_Source, + $field_states: deal.$field_states, + $review_process: deal.$review_process, + Closing_Date: deal.Closing_Date, + Reason_For_Loss__s: deal.Reason_For_Loss__s, + Last_Activity_Time: deal.Last_Activity_Time, + Modified_By: deal.Modified_By, + $review: deal.$review, + Lead_Conversion_Time: deal.Lead_Conversion_Time, + $state: deal.$state, + $process_flow: deal.$process_flow, + Deal_Name: deal.Deal_Name, + Expected_Revenue: deal.Expected_Revenue, + Overall_Sales_Duration: deal.Overall_Sales_Duration, + Stage: deal.Stage, + $locked_for_me: deal.$locked_for_me, + Account_Name: deal.Account_Name, + id: deal.id as string, + $approved: deal.$approved, + $approval: deal.$approval, + Modified_Time: deal.Modified_Time, + Created_Time: deal.Created_Time, + Amount: deal.Amount, + Next_Step: deal.Next_Step, + Probability: deal.Probability, + $editable: deal.$editable, + $orchestration: deal.$orchestration, + Contact_Name: deal.Contact_Name, + Sales_Cycle_Duration: deal.Sales_Cycle_Duration, + Type: deal.Type, + $in_merge: deal.$in_merge, + Locked__s: deal.Locked__s, + Lead_Source: deal.Lead_Source, + Created_By: deal.Created_By, + Tag: deal.Tag, + $zia_owner_assignment: deal.$zia_owner_assignment, + $approval_state: deal.$approval_state + }; +} \ No newline at end of file diff --git a/packages/shared/providers.yaml b/packages/shared/providers.yaml index fc4c65e0106..8d6de1f56bf 100644 --- a/packages/shared/providers.yaml +++ b/packages/shared/providers.yaml @@ -1399,6 +1399,11 @@ zoho: access_type: offline proxy: base_url: https://www.zohoapis.${connectionConfig.extension} + paginate: + type: offset + response_path: data + offset_name_in_request: page + limit_name_in_request: per_page zoho-books: alias: zoho zoho-crm: From f8d887ec56bca42fcb98bb419e7faf0bcd12b644 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 13:55:42 +0100 Subject: [PATCH 33/85] fix: nangoProps lastSyncDate is not decoded as a Date (#1451) using superjson as recommended on trpc documentation to ensure Dates are properly decoded --- package-lock.json | 40 ++++++++++++++++++++++++- packages/jobs/lib/client.ts | 2 ++ packages/jobs/lib/server.ts | 5 +++- packages/jobs/package.json | 3 +- packages/runner/lib/client.ts | 2 ++ packages/runner/lib/client.unit.test.ts | 11 +++++-- packages/runner/lib/server.ts | 5 +++- packages/runner/package.json | 3 +- 8 files changed, 64 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2870d88111c..23f939fc223 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7631,6 +7631,20 @@ "version": "1.0.6", "license": "MIT" }, + "node_modules/copy-anything": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/copy-anything/-/copy-anything-3.0.5.tgz", + "integrity": "sha512-yCEafptTtb4bk7GLEQoM8KVJpxAfdBJYaXyzQEgQQQgYrZiDp8SJmGKlYza6CYjEDNstAdNdKA3UuoULlEbS6w==", + "dependencies": { + "is-what": "^4.1.8" + }, + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/copyfiles": { "version": "2.4.1", "license": "MIT", @@ -9967,6 +9981,17 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-what": { + "version": "4.1.16", + "resolved": "https://registry.npmjs.org/is-what/-/is-what-4.1.16.tgz", + "integrity": "sha512-ZhMwEosbFJkA0YhFnNDgTM4ZxDRsS6HqTo7qsZM08fehyRYIYa0yHu5R6mgo1n/8MgaPBXiPimPD77baVFYg+A==", + "engines": { + "node": ">=12.13" + }, + "funding": { + "url": "https://github.com/sponsors/mesqueeb" + } + }, "node_modules/isarray": { "version": "0.0.1", "license": "MIT" @@ -13206,6 +13231,17 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/superjson": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/superjson/-/superjson-2.2.1.tgz", + "integrity": "sha512-8iGv75BYOa0xRJHK5vRLEjE2H/i4lulTjzpUXic3Eg8akftYjkmQDa8JARQ42rlczXyFR3IeRoeFCc7RxHsYZA==", + "dependencies": { + "copy-anything": "^3.0.2" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/supports-color": { "version": "7.2.0", "license": "MIT", @@ -15333,6 +15369,7 @@ "knex": "^2.4.2", "md5": "^2.3.0", "nanoid": "3.x", + "superjson": "^2.2.1", "uuid": "^9.0.0" }, "devDependencies": { @@ -15702,7 +15739,8 @@ "@nangohq/shared": "0.36.63", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", - "api": "^6.1.1" + "api": "^6.1.1", + "superjson": "^2.2.1" }, "devDependencies": { "@types/node": "^18.7.6", diff --git a/packages/jobs/lib/client.ts b/packages/jobs/lib/client.ts index e875ed6f6a6..71dcac567a3 100644 --- a/packages/jobs/lib/client.ts +++ b/packages/jobs/lib/client.ts @@ -1,8 +1,10 @@ import { CreateTRPCProxyClient, createTRPCProxyClient, httpBatchLink } from '@trpc/client'; import type { AppRouter } from './server.js'; +import superjson from 'superjson'; export function getJobsClient(url: string): CreateTRPCProxyClient { return createTRPCProxyClient({ + transformer: superjson, links: [httpBatchLink({ url })] }); } diff --git a/packages/jobs/lib/server.ts b/packages/jobs/lib/server.ts index 91671409113..2cf0fc593e8 100644 --- a/packages/jobs/lib/server.ts +++ b/packages/jobs/lib/server.ts @@ -1,7 +1,10 @@ import { initTRPC } from '@trpc/server'; import { createHTTPServer } from '@trpc/server/adapters/standalone'; +import superjson from 'superjson'; -const t = initTRPC.create(); +export const t = initTRPC.create({ + transformer: superjson +}); const router = t.router; const publicProcedure = t.procedure; diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 70f9644ecc5..4082c815d97 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -34,6 +34,7 @@ "knex": "^2.4.2", "md5": "^2.3.0", "nanoid": "3.x", + "superjson": "^2.2.1", "uuid": "^9.0.0" }, "devDependencies": { @@ -48,4 +49,4 @@ "nodemon": "^3.0.1", "typescript": "^5.3.2" } -} \ No newline at end of file +} diff --git a/packages/runner/lib/client.ts b/packages/runner/lib/client.ts index 569d6a8d7a7..4ab7888a106 100644 --- a/packages/runner/lib/client.ts +++ b/packages/runner/lib/client.ts @@ -1,8 +1,10 @@ import { CreateTRPCProxyClient, createTRPCProxyClient, httpBatchLink } from '@trpc/client'; import type { AppRouter } from './server.js'; +import superjson from 'superjson'; export function getRunnerClient(url: string): CreateTRPCProxyClient { return createTRPCProxyClient({ + transformer: superjson, links: [httpBatchLink({ url })] }); } diff --git a/packages/runner/lib/client.unit.test.ts b/packages/runner/lib/client.unit.test.ts index a5c86775ad1..50d2e2c8ab6 100644 --- a/packages/runner/lib/client.unit.test.ts +++ b/packages/runner/lib/client.unit.test.ts @@ -35,8 +35,15 @@ describe('Runner client', () => { logMessages: [], stubbedMetadata: {} }; - const jsCode = 'f = () => { return [1, 2, 3] }; exports.default = f'; - const isInvokedImmediately = true; + const jsCode = ` + f = (nango) => { + const s = nango.lastSyncDate.toISOString(); + const b = Buffer.from("hello world"); + return [1, 2, 3] + }; + exports.default = f + `; + const isInvokedImmediately = false; const isWebhook = false; const run = client.run.mutate({ nangoProps, isInvokedImmediately, isWebhook, code: jsCode }); await expect(run).resolves.toEqual([1, 2, 3]); diff --git a/packages/runner/lib/server.ts b/packages/runner/lib/server.ts index eb529fd80a2..9ac56708bc6 100644 --- a/packages/runner/lib/server.ts +++ b/packages/runner/lib/server.ts @@ -2,8 +2,11 @@ import { initTRPC } from '@trpc/server'; import { createHTTPServer } from '@trpc/server/adapters/standalone'; import type { NangoProps } from '@nangohq/shared'; import { exec } from './exec.js'; +import superjson from 'superjson'; -const t = initTRPC.create(); +export const t = initTRPC.create({ + transformer: superjson +}); // const logging = t.middleware(async (opts) => { // // TODO diff --git a/packages/runner/package.json b/packages/runner/package.json index e906812fe4c..fb268370d26 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -21,7 +21,8 @@ "@nangohq/shared": "0.36.63", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", - "api": "^6.1.1" + "api": "^6.1.1", + "superjson": "^2.2.1" }, "devDependencies": { "@types/node": "^18.7.6", From 8fcb636969bb89621920211dd713ef8c39451f1a Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:40:28 +0100 Subject: [PATCH 34/85] 0.36.64 (#1452) --- package-lock.json | 24 ++++++++++++------------ packages/cli/docker/docker-compose.yaml | 4 ++-- packages/cli/package.json | 4 ++-- packages/frontend/package.json | 2 +- packages/jobs/package.json | 2 +- packages/node-client/package.json | 2 +- packages/runner/package.json | 2 +- packages/server/package.json | 6 +++--- packages/shared/package.json | 4 ++-- packages/webapp/package-lock.json | 18 +++++++++--------- packages/webapp/package.json | 4 ++-- packages/worker/package.json | 4 ++-- 12 files changed, 38 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 23f939fc223..39bd77f42da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15237,12 +15237,12 @@ }, "packages/cli": { "name": "nango", - "version": "0.36.63", + "version": "0.36.64", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", @@ -15343,7 +15343,7 @@ }, "packages/frontend": { "name": "@nangohq/frontend", - "version": "0.36.63", + "version": "0.36.64", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY" }, "packages/jobs": { @@ -15351,7 +15351,7 @@ "version": "0.36.52", "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", @@ -15719,7 +15719,7 @@ }, "packages/node-client": { "name": "@nangohq/node", - "version": "0.36.63", + "version": "0.36.64", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "axios": "^1.2.0" @@ -15736,7 +15736,7 @@ "version": "0.36.52", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1", @@ -15762,11 +15762,11 @@ }, "packages/server": { "name": "@nangohq/nango-server", - "version": "0.36.63", + "version": "0.36.64", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -15921,13 +15921,13 @@ }, "packages/shared": { "name": "@nangohq/shared", - "version": "0.36.63", + "version": "0.36.64", "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.63", + "@nangohq/node": "0.36.64", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", @@ -16148,9 +16148,9 @@ }, "packages/worker": { "name": "@nangohq/nango-worker", - "version": "0.36.63", + "version": "0.36.64", "dependencies": { - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 69dfc9462fa..2c9e8c44a5b 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -15,7 +15,7 @@ services: - nango nango-server: - image: nangohq/nango-server:0.36.63 + image: nangohq/nango-server:0.36.64 container_name: nango-server environment: - TEMPORAL_ADDRESS=temporal:7233 @@ -47,7 +47,7 @@ services: - nango nango-worker: - image: nangohq/nango-worker:0.36.63 + image: nangohq/nango-worker:0.36.64 container_name: nango-worker restart: always ports: diff --git a/packages/cli/package.json b/packages/cli/package.json index 39e3f131344..670934f51d9 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "nango", - "version": "0.36.63", + "version": "0.36.64", "description": "Nango's CLI tool.", "type": "module", "main": "dist/index.js", @@ -23,7 +23,7 @@ "dependencies": { "@babel/traverse": "^7.22.5", "@inquirer/prompts": "^2.3.0", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@vercel/ncc": "^0.36.1", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 1c52df6b5fe..df188558059 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/frontend", - "version": "0.36.63", + "version": "0.36.64", "description": "Nango's frontend library for OAuth handling.", "type": "module", "main": "dist/index.js", diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 4082c815d97..300d5838b08 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@nangohq/nango-runner": "0.36.52", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", diff --git a/packages/node-client/package.json b/packages/node-client/package.json index 83016396bdc..52fbbe597c7 100644 --- a/packages/node-client/package.json +++ b/packages/node-client/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/node", - "version": "0.36.63", + "version": "0.36.64", "description": "Nango's Node client.", "type": "module", "main": "dist/index.js", diff --git a/packages/runner/package.json b/packages/runner/package.json index fb268370d26..18dfd868ada 100644 --- a/packages/runner/package.json +++ b/packages/runner/package.json @@ -18,7 +18,7 @@ }, "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", "dependencies": { - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@trpc/client": "^10.44.0", "@trpc/server": "^10.44.0", "api": "^6.1.1", diff --git a/packages/server/package.json b/packages/server/package.json index 61a14bc3c56..01f9c8f1ecc 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-server", - "version": "0.36.63", + "version": "0.36.64", "description": "Nango OAuth's server.", "type": "module", "main": "dist/server.js", @@ -22,7 +22,7 @@ }, "dependencies": { "@hapi/boom": "^10.0.1", - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.7.4", "axios": "^1.3.4", @@ -74,4 +74,4 @@ "nodemon": "^3.0.1", "typescript": "^4.7.4" } -} \ No newline at end of file +} diff --git a/packages/shared/package.json b/packages/shared/package.json index b463e1a52e0..4236980b7c2 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/shared", - "version": "0.36.63", + "version": "0.36.64", "description": "Nango's shared components.", "type": "module", "main": "dist/lib/index.js", @@ -19,7 +19,7 @@ "@aws-sdk/client-s3": "^3.348.0", "@datadog/datadog-api-client": "^1.16.0", "@hapi/boom": "^10.0.1", - "@nangohq/node": "0.36.63", + "@nangohq/node": "0.36.64", "@sentry/node": "^7.37.2", "@temporalio/client": "^1.5.2", "@types/fs-extra": "^11.0.1", diff --git a/packages/webapp/package-lock.json b/packages/webapp/package-lock.json index aab9888e5c5..9d6d55a2d9c 100644 --- a/packages/webapp/package-lock.json +++ b/packages/webapp/package-lock.json @@ -1,19 +1,19 @@ { "name": "webapp", - "version": "0.36.63", + "version": "0.36.64", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "webapp", - "version": "0.36.63", + "version": "0.36.64", "dependencies": { "@geist-ui/core": "^2.3.8", "@geist-ui/icons": "^1.0.2", "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.63", + "@nangohq/frontend": "0.36.64", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", @@ -3120,9 +3120,9 @@ } }, "node_modules/@nangohq/frontend": { - "version": "0.36.63", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.63.tgz", - "integrity": "sha512-A2UfIbu63lqDMefsn+EzOZ01Z5rLX0v/G3vA3N2uRUnWq+u97K+NdIpe4y4ckiPz4GrcO6mIiobVuSkFpqo72w==" + "version": "0.36.64", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.64.tgz", + "integrity": "sha512-j8Vh/4NA4xRbfjjqhDpGEXQCDd8yHnDhkQ6xCAO5UsPPUwpBdjaBv32DDdyxHV/vmxkMKhwQDZVoygPssGhCHA==" }, "node_modules/@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", @@ -18895,9 +18895,9 @@ "requires": {} }, "@nangohq/frontend": { - "version": "0.36.63", - "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.63.tgz", - "integrity": "sha512-A2UfIbu63lqDMefsn+EzOZ01Z5rLX0v/G3vA3N2uRUnWq+u97K+NdIpe4y4ckiPz4GrcO6mIiobVuSkFpqo72w==" + "version": "0.36.64", + "resolved": "https://registry.npmjs.org/@nangohq/frontend/-/frontend-0.36.64.tgz", + "integrity": "sha512-j8Vh/4NA4xRbfjjqhDpGEXQCDd8yHnDhkQ6xCAO5UsPPUwpBdjaBv32DDdyxHV/vmxkMKhwQDZVoygPssGhCHA==" }, "@nicolo-ribaudo/eslint-scope-5-internals": { "version": "5.1.1-v1", diff --git a/packages/webapp/package.json b/packages/webapp/package.json index 16b02900585..ecd9fb0979c 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -1,6 +1,6 @@ { "name": "webapp", - "version": "0.36.63", + "version": "0.36.64", "private": true, "dependencies": { "@geist-ui/core": "^2.3.8", @@ -8,7 +8,7 @@ "@headlessui/react": "^1.7.12", "@heroicons/react": "^2.0.18", "@mantine/prism": "^5.10.5", - "@nangohq/frontend": "0.36.63", + "@nangohq/frontend": "0.36.64", "@sentry/react": "^7.83.0", "@testing-library/jest-dom": "^5.16.5", "@testing-library/react": "^13.4.0", diff --git a/packages/worker/package.json b/packages/worker/package.json index 7536b2a758a..8a67a72a0ee 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,6 +1,6 @@ { "name": "@nangohq/nango-worker", - "version": "0.36.63", + "version": "0.36.64", "type": "module", "main": "dist/worker.js", "scripts": { @@ -15,7 +15,7 @@ "directory": "packages/worker" }, "dependencies": { - "@nangohq/shared": "0.36.63", + "@nangohq/shared": "0.36.64", "@octokit/plugin-retry": "^6.0.0", "@octokit/plugin-throttling": "^7.0.0", "@octokit/rest": "^20.0.1", From 8a1e8653d5d72d48b0eb2e996cd5835cf68b69f3 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:47:59 +0100 Subject: [PATCH 35/85] Hide postgress error internal details (#1408) * postgress error doesn't always have detail Not showing `Detail: undefined` when no error detail are returned by postgres. Otherwise it is confusing and misleading. * data.service: do not return internal Postgres error details --- .../shared/lib/services/sync/data/data.service.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/shared/lib/services/sync/data/data.service.ts b/packages/shared/lib/services/sync/data/data.service.ts index 431644fd593..2910db714a6 100644 --- a/packages/shared/lib/services/sync/data/data.service.ts +++ b/packages/shared/lib/services/sync/data/data.service.ts @@ -85,10 +85,18 @@ export async function upsert( errorMessage += `Model: ${model}, Unique Key: ${uniqueKey}, Nango Connection ID: ${nangoConnectionId}.\n`; errorMessage += `Attempted to insert/update/delete: ${responseWithoutDuplicates.length} records\n`; - if ('code' in error) errorMessage += `Error code: ${error.code}.\n`; - if ('detail' in error) errorMessage += `Detail: ${error.detail}.\n`; + if (error.code) errorMessage += `Error code: ${error.code}.\n`; - errorMessage += `Error Message: ${error.message}`; + console.log(`${errorMessage}${error}`); + + let errorDetail = ''; + switch (error.code) { + case '22001': { + errorDetail = 'Payload too big. Limit = 256MB'; + break; + } + } + if (errorDetail) errorMessage += `Info: ${errorDetail}.\n`; return { success: false, From ac0ef2c97648e2b0286d0e774e5e69015c9d81fd Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:53:28 +0100 Subject: [PATCH 36/85] Remove nango worker (#1439) --- .github/workflows/docker.yaml | 8 +- package-lock.json | 221 +---------- package.json | 8 +- packages/cli/docker/docker-compose.yaml | 8 +- packages/cli/lib/cli.ts | 3 - packages/cli/lib/index.ts | 2 +- packages/worker/.gitignore | 3 - packages/worker/Dockerfile | 19 - packages/worker/lib/activities.ts | 408 --------------------- packages/worker/lib/integration.service.ts | 183 --------- packages/worker/lib/models/Worker.ts | 34 -- packages/worker/lib/worker.ts | 61 --- packages/worker/lib/workflows.ts | 56 --- packages/worker/nodemon.json | 6 - packages/worker/package.json | 48 --- packages/worker/tsconfig.json | 9 - scripts/beta-release.bash | 108 ------ scripts/release.bash | 26 +- tsconfig.build.json | 1 - 19 files changed, 22 insertions(+), 1190 deletions(-) delete mode 100644 packages/worker/.gitignore delete mode 100644 packages/worker/Dockerfile delete mode 100644 packages/worker/lib/activities.ts delete mode 100644 packages/worker/lib/integration.service.ts delete mode 100644 packages/worker/lib/models/Worker.ts delete mode 100644 packages/worker/lib/worker.ts delete mode 100644 packages/worker/lib/workflows.ts delete mode 100644 packages/worker/nodemon.json delete mode 100644 packages/worker/package.json delete mode 100644 packages/worker/tsconfig.json delete mode 100755 scripts/beta-release.bash diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml index 45e320a2763..87751ec0e04 100644 --- a/.github/workflows/docker.yaml +++ b/.github/workflows/docker.yaml @@ -37,10 +37,10 @@ jobs: exit 1 fi - CONTAINER_NAME=nango-worker - WORKER=$(docker ps -q -f status=running -f name=^/${CONTAINER_NAME}$) - if [ ! "${WORKER}" ]; then - echo "Worker container doesn't exist" + CONTAINER_NAME=nango-jobs + JOBS=$(docker ps -q -f status=running -f name=^/${CONTAINER_NAME}$) + if [ ! "${JOBS}" ]; then + echo "Jobs container doesn't exist" exit 1 fi shell: bash diff --git a/package-lock.json b/package-lock.json index 39bd77f42da..50b26afebfd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,6 @@ "packages/frontend", "packages/node-client", "packages/server", - "packages/worker", "packages/runner", "packages/jobs" ], @@ -3731,10 +3730,6 @@ "resolved": "packages/server", "link": true }, - "node_modules/@nangohq/nango-worker": { - "resolved": "packages/worker", - "link": true - }, "node_modules/@nangohq/node": { "resolved": "packages/node-client", "link": true @@ -14791,20 +14786,6 @@ "resolved": "https://registry.npmjs.org/vm/-/vm-0.1.0.tgz", "integrity": "sha512-1aKVjgohVDnVhGrJLhl2k8zIrapH+7HsdnIjGvBp3XX2OCj6XGzsIbDp9rZ3r7t6qgDfXEE1EoEAEOLJm9LKnw==" }, - "node_modules/vm2": { - "version": "3.9.19", - "license": "MIT", - "dependencies": { - "acorn": "^8.7.0", - "acorn-walk": "^8.2.0" - }, - "bin": { - "vm2": "bin/vm2" - }, - "engines": { - "node": ">=6.0" - } - }, "node_modules/watchpack": { "version": "2.4.0", "license": "MIT", @@ -16149,6 +16130,7 @@ "packages/worker": { "name": "@nangohq/nango-worker", "version": "0.36.64", + "extraneous": true, "dependencies": { "@nangohq/shared": "0.36.64", "@octokit/plugin-retry": "^6.0.0", @@ -16180,207 +16162,6 @@ "nodemon": "^3.0.1", "typescript": "^4.4.2" } - }, - "packages/worker/node_modules/@eslint/eslintrc": { - "version": "0.4.3", - "dev": true, - "license": "MIT", - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.1.1", - "espree": "^7.3.0", - "globals": "^13.9.0", - "ignore": "^4.0.6", - "import-fresh": "^3.2.1", - "js-yaml": "^3.13.1", - "minimatch": "^3.0.4", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "packages/worker/node_modules/@eslint/eslintrc/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "packages/worker/node_modules/@humanwhocodes/config-array": { - "version": "0.5.0", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@humanwhocodes/object-schema": "^1.2.0", - "debug": "^4.1.1", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "packages/worker/node_modules/acorn": { - "version": "7.4.1", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "packages/worker/node_modules/argparse": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "packages/worker/node_modules/eslint": { - "version": "7.32.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "7.12.11", - "@eslint/eslintrc": "^0.4.3", - "@humanwhocodes/config-array": "^0.5.0", - "ajv": "^6.10.0", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.0.1", - "doctrine": "^3.0.0", - "enquirer": "^2.3.5", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^5.1.1", - "eslint-utils": "^2.1.0", - "eslint-visitor-keys": "^2.0.0", - "espree": "^7.3.1", - "esquery": "^1.4.0", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "functional-red-black-tree": "^1.0.1", - "glob-parent": "^5.1.2", - "globals": "^13.6.0", - "ignore": "^4.0.6", - "import-fresh": "^3.0.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "js-yaml": "^3.13.1", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.0.4", - "natural-compare": "^1.4.0", - "optionator": "^0.9.1", - "progress": "^2.0.0", - "regexpp": "^3.1.0", - "semver": "^7.2.1", - "strip-ansi": "^6.0.0", - "strip-json-comments": "^3.1.0", - "table": "^6.0.9", - "text-table": "^0.2.0", - "v8-compile-cache": "^2.0.3" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "packages/worker/node_modules/eslint-utils": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "eslint-visitor-keys": "^1.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/mysticatea" - } - }, - "packages/worker/node_modules/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "1.3.0", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=4" - } - }, - "packages/worker/node_modules/eslint-visitor-keys": { - "version": "2.1.0", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=10" - } - }, - "packages/worker/node_modules/eslint/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "packages/worker/node_modules/espree": { - "version": "7.3.1", - "dev": true, - "license": "BSD-2-Clause", - "dependencies": { - "acorn": "^7.4.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^1.3.0" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "packages/worker/node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "1.3.0", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=4" - } - }, - "packages/worker/node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "packages/worker/node_modules/ignore": { - "version": "4.0.6", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4" - } } } } diff --git a/package.json b/package.json index 96f4e469177..18aced87d9d 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,6 @@ "packages/frontend", "packages/node-client", "packages/server", - "packages/worker", "packages/runner", "packages/jobs" ], @@ -19,7 +18,7 @@ "lint": "eslint . --ext .ts,.tsx", "lint:fix": "eslint . --ext .ts,.tsx --fix", "install:all": "npm i && cd packages/webapp && npm i && cd ../..", - "ts-build": "tsc -b --clean packages/shared packages/server packages/worker packages/cli packages/runner packages/jobs && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json", + "ts-build": "tsc -b --clean packages/shared packages/server packages/cli packages/runner packages/jobs && tsc -b tsconfig.build.json && tsc -b packages/webapp/tsconfig.json", "docker-build": "docker build -f packages/server/Dockerfile -t nango-server:latest .", "webapp-build:hosted": "cd ./packages/webapp && npm run build:hosted && cd ../..", "webapp-build:staging": "cd ./packages/webapp && npm run build:staging && cd ../..", @@ -31,7 +30,6 @@ "build:staging": "npm run install:all && npm run ts-build && npm run webapp-build:staging", "build:prod": "npm run install:all && npm run ts-build && npm run webapp-build:prod", "server:dev:watch": "cd ./packages/server && npm run dev", - "worker:dev:watch": "cd ./packages/worker && npm run dev", "jobs:dev:watch": "npm run dev -w @nangohq/nango-jobs", "webapp:dev:watch": "cd ./packages/webapp && npm run start:hosted", "cli:watch": "cd ./packages/cli && npm run dev", @@ -39,7 +37,7 @@ "build:watch": "tsc -b -w tsconfig.build.json", "dev:watch": "concurrently --kill-others \"npm run build:watch\" \"npm run webapp-build:watch\" \"npm run prettier-watch\" \"npm run shared:watch\" \"npm run cli:watch\"", "watch:dev": "npm run dev:watch", - "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run worker:dev:watch\" \"npm run jobs:dev:watch\"", + "dev:watch:apps": "concurrently --kill-others \"npm run server:dev:watch\" \"npm run webapp:dev:watch\" \"npm run jobs:dev:watch\"", "watch:dev:apps": "npm run dev:watch:apps", "dw": "npm run dev:watch", "dwa": "npm run dev:watch:apps", @@ -71,4 +69,4 @@ "@babel/parser": "^7.22.5", "@babel/traverse": "^7.22.5" } -} +} \ No newline at end of file diff --git a/packages/cli/docker/docker-compose.yaml b/packages/cli/docker/docker-compose.yaml index 2c9e8c44a5b..424db69822c 100644 --- a/packages/cli/docker/docker-compose.yaml +++ b/packages/cli/docker/docker-compose.yaml @@ -46,9 +46,9 @@ services: networks: - nango - nango-worker: - image: nangohq/nango-worker:0.36.64 - container_name: nango-worker + nango-jobs: + image: nangohq/nango-jobs:production + container_name: nango-jobs restart: always ports: - '${WORKER_PORT:-3004}:${WORKER_PORT:-3004}' @@ -64,7 +64,7 @@ services: - nango-db - temporal volumes: - - "${NANGO_INTEGRATIONS_LOCATION:-./}:/usr/nango-worker/src/packages/shared/dist/lib/nango-integrations" + - "${NANGO_INTEGRATIONS_LOCATION:-./}:/usr/nango-jobs/src/packages/shared/dist/lib/nango-integrations" networks: - nango diff --git a/packages/cli/lib/cli.ts b/packages/cli/lib/cli.ts index d5de95ebbe4..9842998917b 100644 --- a/packages/cli/lib/cli.ts +++ b/packages/cli/lib/cli.ts @@ -33,13 +33,10 @@ export const version = (debug: boolean) => { const dockerCompose = yaml.load(dockerComposeYaml) as any; const nangoServerImage = dockerCompose.services['nango-server'].image; - const nangoWorkerImage = dockerCompose.services['nango-worker'].image; const nangoServerVersion = nangoServerImage.split(':').pop(); - const nangoWorkerVersion = nangoWorkerImage.split(':').pop(); console.log(chalk.green('Nango Server version:'), nangoServerVersion); - console.log(chalk.green('Nango Worker version:'), nangoWorkerVersion); console.log(chalk.green('Nango CLI version:'), packageJson.version); }; diff --git a/packages/cli/lib/index.ts b/packages/cli/lib/index.ts index f529f3ffc47..0e146981d6e 100644 --- a/packages/cli/lib/index.ts +++ b/packages/cli/lib/index.ts @@ -76,7 +76,7 @@ program.addHelpText('before', chalk.green(figlet.textSync('Nango CLI'))); program .command('version') - .description('Print the version of the Nango CLI, Nango Worker, and Nango Server.') + .description('Print the version of the Nango CLI and Nango Server.') .action(function (this: Command) { const { debug } = this.opts(); version(debug); diff --git a/packages/worker/.gitignore b/packages/worker/.gitignore deleted file mode 100644 index 8c3aba0aad2..00000000000 --- a/packages/worker/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -tsconfig.tsbuildinfo -dist/* -node_modules diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile deleted file mode 100644 index c42952c6f56..00000000000 --- a/packages/worker/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -FROM node:18-slim - -RUN apt-get update \ - && apt-get install -y ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -ENV SERVER_RUN_MODE=DOCKERIZED - -WORKDIR /usr/nango-worker/src - -COPY packages/shared/ packages/shared/ -COPY packages/node-client/ packages/node-client/ -COPY packages/worker/ packages/worker/ -COPY package*.json ./ - -RUN npm pkg delete scripts.prepare -RUN npm install --omit=dev - -CMD [ "node", "packages/worker/dist/worker.js" ] diff --git a/packages/worker/lib/activities.ts b/packages/worker/lib/activities.ts deleted file mode 100644 index d7b91cd831d..00000000000 --- a/packages/worker/lib/activities.ts +++ /dev/null @@ -1,408 +0,0 @@ -import { Context, CancelledFailure } from '@temporalio/activity'; -import { TimeoutFailure, TerminatedFailure } from '@temporalio/client'; -import { - createSyncJob, - SyncStatus, - SyncType, - Config as ProviderConfig, - configService, - createActivityLog, - LogLevel, - LogActionEnum, - syncRunService, - ServiceResponse, - NangoConnection, - environmentService, - createActivityLogMessage, - createActivityLogAndLogMessage, - ErrorSourceEnum, - errorManager, - metricsManager, - updateSyncJobStatus, - updateLatestJobSyncStatus, - MetricTypes, - isInitialSyncStillRunning, - initialSyncExists, - getSyncByIdAndName, - logger -} from '@nangohq/shared'; -import integrationService from './integration.service.js'; -import type { WebhookArgs, ContinuousSyncArgs, InitialSyncArgs, ActionArgs } from './models/Worker'; - -export async function routeSync(args: InitialSyncArgs): Promise { - const { syncId, syncJobId, syncName, activityLogId, nangoConnection, debug } = args; - let environmentId = nangoConnection?.environment_id; - - // https://typescript.temporal.io/api/classes/activity.Context - const context: Context = Context.current(); - if (!nangoConnection?.environment_id) { - environmentId = (await environmentService.getEnvironmentIdForAccountAssumingProd(nangoConnection.account_id as number)) as number; - } - const syncConfig: ProviderConfig = (await configService.getProviderConfig(nangoConnection?.provider_config_key as string, environmentId)) as ProviderConfig; - - return syncProvider( - syncConfig, - syncId, - syncJobId, - syncName, - SyncType.INITIAL, - { ...nangoConnection, environment_id: environmentId }, - activityLogId, - context, - debug - ); -} - -export async function runAction(args: ActionArgs): Promise { - const { input, nangoConnection, actionName, activityLogId } = args; - - const syncConfig: ProviderConfig = (await configService.getProviderConfig( - nangoConnection?.provider_config_key as string, - nangoConnection?.environment_id as number - )) as ProviderConfig; - - const context: Context = Context.current(); - - const syncRun = new syncRunService({ - integrationService, - writeToDb: true, - nangoConnection, - syncName: actionName, - isAction: true, - syncType: SyncType.ACTION, - activityLogId, - input, - provider: syncConfig.provider, - debug: false, - temporalContext: context - }); - - const actionResults = await syncRun.run(); - - return actionResults; -} - -export async function runWebhook(args: WebhookArgs): Promise { - const { input, nangoConnection, activityLogId, parentSyncName } = args; - - const syncConfig: ProviderConfig = (await configService.getProviderConfig( - nangoConnection?.provider_config_key as string, - nangoConnection?.environment_id as number - )) as ProviderConfig; - - const sync = await getSyncByIdAndName(nangoConnection.id as number, parentSyncName); - - const context: Context = Context.current(); - - const syncJobId = await createSyncJob( - sync?.id as string, - SyncType.WEBHOOK, - SyncStatus.RUNNING, - context.info.workflowExecution.workflowId, - nangoConnection, - context.info.workflowExecution.runId - ); - - const syncRun = new syncRunService({ - integrationService, - writeToDb: true, - nangoConnection, - syncJobId: syncJobId?.id as number, - syncName: parentSyncName, - isAction: false, - syncType: SyncType.WEBHOOK, - isWebhook: true, - activityLogId, - input, - provider: syncConfig.provider, - debug: false, - temporalContext: context - }); - - const result = await syncRun.run(); - - return result.success; -} - -export async function scheduleAndRouteSync(args: ContinuousSyncArgs): Promise { - const { syncId, activityLogId, syncName, nangoConnection, debug } = args; - let environmentId = nangoConnection?.environment_id; - let syncJobId; - - const initialSyncStillRunning = await isInitialSyncStillRunning(syncId as string); - - if (initialSyncStillRunning) { - const content = `The continuous sync "${syncName}" with sync id ${syncId} did not run because the initial sync is still running. It will attempt to run at the next scheduled time.`; - - logger.log('info', content); - - await metricsManager.capture(MetricTypes.SYNC_OVERLAP, content, LogActionEnum.SYNC, { - environmentId: String(nangoConnection?.environment_id), - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - syncName, - syncId - }); - - return true; - } - - // https://typescript.temporal.io/api/classes/activity.Context - const context: Context = Context.current(); - const syncType = (await initialSyncExists(syncId as string)) ? SyncType.INCREMENTAL : SyncType.INITIAL; - try { - if (!nangoConnection?.environment_id) { - environmentId = (await environmentService.getEnvironmentIdForAccountAssumingProd(nangoConnection.account_id as number)) as number; - syncJobId = await createSyncJob( - syncId as string, - syncType, - SyncStatus.RUNNING, - context.info.workflowExecution.workflowId, - nangoConnection, - context.info.workflowExecution.runId - ); - } else { - syncJobId = await createSyncJob( - syncId as string, - syncType, - SyncStatus.RUNNING, - context.info.workflowExecution.workflowId, - nangoConnection, - context.info.workflowExecution.runId - ); - } - - const syncConfig: ProviderConfig = (await configService.getProviderConfig( - nangoConnection?.provider_config_key as string, - environmentId - )) as ProviderConfig; - - return syncProvider( - syncConfig, - syncId, - syncJobId?.id as number, - syncName, - syncType, - { ...nangoConnection, environment_id: environmentId }, - activityLogId ?? 0, - context, - debug - ); - } catch (err: any) { - const prettyError = JSON.stringify(err, ['message', 'name'], 2); - const log = { - level: 'info' as LogLevel, - success: false, - action: LogActionEnum.SYNC, - start: Date.now(), - end: Date.now(), - timestamp: Date.now(), - connection_id: nangoConnection?.connection_id as string, - provider_config_key: nangoConnection?.provider_config_key as string, - provider: '', - session_id: '', - environment_id: environmentId, - operation_name: syncName - }; - const content = `The continuous sync failed to run because of a failure to obtain the provider config for ${syncName} with the following error: ${prettyError}`; - await createActivityLogAndLogMessage(log, { - level: 'error', - environment_id: environmentId, - timestamp: Date.now(), - content - }); - - await metricsManager.capture(MetricTypes.SYNC_FAILURE, content, LogActionEnum.SYNC, { - environmentId: String(environmentId), - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - syncId, - syncName - }); - - await errorManager.report(content, { - environmentId, - source: ErrorSourceEnum.PLATFORM, - operation: LogActionEnum.SYNC, - metadata: { - syncType, - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - syncName - } - }); - - return false; - } -} - -/** - * Sync Provider - * @desc take in a provider, use the nango.yaml config to find - * the integrations where that provider is used and call the sync - * accordingly with the user defined integration code - */ -export async function syncProvider( - syncConfig: ProviderConfig, - syncId: string, - syncJobId: number, - syncName: string, - syncType: SyncType, - nangoConnection: NangoConnection, - existingActivityLogId: number, - temporalContext: Context, - debug = false -): Promise { - try { - let activityLogId = existingActivityLogId; - - if (syncType === SyncType.INCREMENTAL || existingActivityLogId === 0) { - const log = { - level: 'info' as LogLevel, - success: null, - action: LogActionEnum.SYNC, - start: Date.now(), - end: Date.now(), - timestamp: Date.now(), - connection_id: nangoConnection?.connection_id as string, - provider_config_key: nangoConnection?.provider_config_key as string, - provider: syncConfig.provider, - session_id: syncJobId ? syncJobId?.toString() : '', - environment_id: nangoConnection?.environment_id as number, - operation_name: syncName - }; - activityLogId = (await createActivityLog(log)) as number; - } - - if (debug) { - await createActivityLogMessage({ - level: 'info', - environment_id: nangoConnection?.environment_id as number, - activity_log_id: activityLogId, - timestamp: Date.now(), - content: `Starting sync ${syncType} for ${syncName} with syncId ${syncId} and syncJobId ${syncJobId} with execution id of ${temporalContext.info.workflowExecution.workflowId} for attempt #${temporalContext.info.attempt}` - }); - } - - const syncRun = new syncRunService({ - integrationService, - writeToDb: true, - syncId, - syncJobId, - nangoConnection, - syncName, - syncType, - activityLogId, - provider: syncConfig.provider, - temporalContext, - debug - }); - - const result = await syncRun.run(); - - return result.response; - } catch (err: any) { - const prettyError = JSON.stringify(err, ['message', 'name'], 2); - const log = { - level: 'info' as LogLevel, - success: false, - action: LogActionEnum.SYNC, - start: Date.now(), - end: Date.now(), - timestamp: Date.now(), - connection_id: nangoConnection?.connection_id as string, - provider_config_key: nangoConnection?.provider_config_key as string, - provider: syncConfig.provider, - session_id: syncJobId ? syncJobId?.toString() : '', - environment_id: nangoConnection?.environment_id as number, - operation_name: syncName - }; - const content = `The ${syncType} sync failed to run because of a failure to create the job and run the sync with the error: ${prettyError}`; - - await createActivityLogAndLogMessage(log, { - level: 'error', - environment_id: nangoConnection?.environment_id as number, - timestamp: Date.now(), - content - }); - - await metricsManager.capture(MetricTypes.SYNC_OVERLAP, content, LogActionEnum.SYNC, { - environmentId: String(nangoConnection?.environment_id), - syncId, - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - syncName - }); - - await errorManager.report(content, { - environmentId: nangoConnection?.environment_id as number, - source: ErrorSourceEnum.PLATFORM, - operation: LogActionEnum.SYNC, - metadata: { - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - syncType, - syncName - } - }); - - return false; - } -} - -export async function reportFailure( - error: any, - workflowArguments: InitialSyncArgs | ContinuousSyncArgs | ActionArgs | WebhookArgs, - DEFAULT_TIMEOUT: string, - MAXIMUM_ATTEMPTS: number -): Promise { - const { nangoConnection } = workflowArguments; - let type = 'webhook'; - - let name = ''; - if ('syncName' in workflowArguments) { - name = workflowArguments.syncName; - type = 'sync'; - } else if ('actionName' in workflowArguments) { - name = workflowArguments.actionName; - type = 'action'; - } else { - name = workflowArguments.name; - } - - let content = `The ${type} "${name}" failed `; - const context: Context = Context.current(); - - if (error instanceof CancelledFailure) { - content += `due to a cancellation.`; - } else if (error.cause instanceof TerminatedFailure || error.cause.name === 'TerminatedFailure') { - content += `due to a termination.`; - } else if (error.cause instanceof TimeoutFailure || error.cause.name === 'TimeoutFailure') { - if (error.cause.timeoutType === 3) { - content += `due to a timeout with respect to the max schedule length timeout of ${DEFAULT_TIMEOUT}.`; - } else { - content += `due to a timeout and a lack of heartbeat with ${MAXIMUM_ATTEMPTS} attempts.`; - } - } else { - content += `due to a unknown failure.`; - } - - await metricsManager.capture(MetricTypes.FLOW_JOB_TIMEOUT_FAILURE, content, LogActionEnum.SYNC, { - environmentId: String(nangoConnection?.environment_id), - name, - connectionId: nangoConnection?.connection_id as string, - providerConfigKey: nangoConnection?.provider_config_key as string, - error: JSON.stringify(error), - info: JSON.stringify(context.info), - workflowId: context.info.workflowExecution.workflowId, - runId: context.info.workflowExecution.runId - }); - - if (type === 'sync' && 'syncId' in workflowArguments) { - if ('syncJobId' in workflowArguments) { - await updateSyncJobStatus(workflowArguments.syncJobId, SyncStatus.STOPPED); - } else { - await updateLatestJobSyncStatus(workflowArguments.syncId, SyncStatus.STOPPED); - } - } -} diff --git a/packages/worker/lib/integration.service.ts b/packages/worker/lib/integration.service.ts deleted file mode 100644 index 98d10a4969c..00000000000 --- a/packages/worker/lib/integration.service.ts +++ /dev/null @@ -1,183 +0,0 @@ -import type { Context } from '@temporalio/activity'; -import { NodeVM } from 'vm2'; -import { - IntegrationServiceInterface, - createActivityLogMessage, - getRootDir, - NangoIntegrationData, - NangoProps, - NangoAction, - NangoSync, - localFileService, - remoteFileService, - isCloud, - ServiceResponse, - NangoError, - formatScriptError -} from '@nangohq/shared'; - -class IntegrationService implements IntegrationServiceInterface { - public runningScripts: { [key: string]: Context } = {}; - - constructor() { - this.sendHeartbeat(); - } - - async runScript( - syncName: string, - syncId: string, - activityLogId: number | undefined, - nangoProps: NangoProps, - integrationData: NangoIntegrationData, - environmentId: number, - writeToDb: boolean, - isInvokedImmediately: boolean, - isWebhook: boolean, - optionalLoadLocation?: string, - input?: object, - temporalContext?: Context - ): Promise> { - try { - const nango = isInvokedImmediately && !isWebhook ? new NangoAction(nangoProps) : new NangoSync(nangoProps); - const script: string | null = - isCloud() && !optionalLoadLocation - ? await remoteFileService.getFile(integrationData.fileLocation as string, environmentId) - : localFileService.getIntegrationFile(syncName, optionalLoadLocation); - - if (!script) { - const content = `Unable to find integration file for ${syncName}`; - - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content, - timestamp: Date.now() - }); - } - } - - if (!script && activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content: `Unable to find integration file for ${syncName}`, - timestamp: Date.now() - }); - - const error = new NangoError('Unable to find integration file', 404); - - return { success: false, error, response: null }; - } - - try { - if (temporalContext) { - this.runningScripts[syncId] = temporalContext; - } - - const vm = new NodeVM({ - console: 'inherit', - sandbox: { nango }, - require: { - external: true, - builtin: ['url', 'crypto'] - } - }); - - const rootDir = getRootDir(optionalLoadLocation); - const scriptExports = vm.run(script as string, `${rootDir}/*.js`); - - if (isWebhook) { - if (!scriptExports.onWebhookPayloadReceived) { - const content = `There is no onWebhookPayloadReceived export for ${syncName}`; - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content, - timestamp: Date.now() - }); - } - - return { success: false, error: new NangoError(content, 500), response: null }; - } - - const results = await scriptExports.onWebhookPayloadReceived(nango, input); - - return { success: true, error: null, response: results }; - } else { - if (typeof scriptExports.default === 'function') { - const results = isInvokedImmediately ? await scriptExports.default(nango, input) : await scriptExports.default(nango); - - return { success: true, error: null, response: results }; - } else { - const content = `There is no default export that is a function for ${syncName}`; - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content, - timestamp: Date.now() - }); - } - - return { success: false, error: new NangoError(content, 500), response: null }; - } - } - } catch (err: any) { - let errorType = 'sync_script_failure'; - if (isWebhook) { - errorType = 'webhook_script_failure'; - } else if (isInvokedImmediately) { - errorType = 'action_script_failure'; - } - const { success, error, response } = formatScriptError(err, errorType, syncName); - - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content: error.message, - timestamp: Date.now() - }); - } - - return { success, error, response }; - } - } catch (err) { - const errorMessage = JSON.stringify(err, ['message', 'name', 'stack'], 2); - const content = `The script failed to load for ${syncName} with the following error: ${errorMessage}`; - - if (activityLogId && writeToDb) { - await createActivityLogMessage({ - level: 'error', - environment_id: environmentId, - activity_log_id: activityLogId, - content, - timestamp: Date.now() - }); - } - - return { success: false, error: new NangoError(content, 500), response: null }; - } finally { - delete this.runningScripts[syncId]; - } - } - - private sendHeartbeat() { - setInterval(() => { - Object.keys(this.runningScripts).forEach((syncId) => { - const context = this.runningScripts[syncId]; - - context?.heartbeat(); - }); - }, 300000); - } -} - -export default new IntegrationService(); diff --git a/packages/worker/lib/models/Worker.ts b/packages/worker/lib/models/Worker.ts deleted file mode 100644 index 3cae8d7a052..00000000000 --- a/packages/worker/lib/models/Worker.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { NangoConnection, NangoIntegrationData } from '@nangohq/shared'; - -export interface InitialSyncArgs { - syncId: string; - syncJobId: number; - syncName: string; - activityLogId: number; - nangoConnection: NangoConnection; - debug?: boolean; -} - -export interface ContinuousSyncArgs { - syncId: string; - activityLogId: number; - syncName: string; - syncData: NangoIntegrationData; - nangoConnection: NangoConnection; - debug?: boolean; -} - -export interface ActionArgs { - input: object; - actionName: string; - nangoConnection: NangoConnection; - activityLogId: number; -} - -export interface WebhookArgs { - name: string; - parentSyncName: string; - nangoConnection: NangoConnection; - input: object; - activityLogId: number; -} diff --git a/packages/worker/lib/worker.ts b/packages/worker/lib/worker.ts deleted file mode 100644 index f29d347eb96..00000000000 --- a/packages/worker/lib/worker.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { Worker, NativeConnection } from '@temporalio/worker'; -import fs from 'fs-extra'; -import * as dotenv from 'dotenv'; -import { createRequire } from 'module'; -import * as activities from './activities.js'; -import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd } from '@nangohq/shared'; - -async function run() { - if (process.env['SERVER_RUN_MODE'] !== 'DOCKERIZED') { - dotenv.config({ path: '../../.env' }); - } - - let crt: Buffer | null = null; - let key: Buffer | null = null; - - const namespace = process.env['TEMPORAL_NAMESPACE'] || 'default'; - - if (isProd()) { - crt = await fs.readFile(`/etc/secrets/${namespace}.crt`); - key = await fs.readFile(`/etc/secrets/${namespace}.key`); - } - - const connection = await NativeConnection.connect({ - address: process.env['TEMPORAL_ADDRESS'] || 'localhost:7233', - tls: !isProd() - ? false - : { - clientCertPair: { - crt: crt as Buffer, - key: key as Buffer - } - } - }); - - const syncWorker = { - connection, - namespace, - workflowsPath: createRequire(import.meta.url).resolve('./workflows'), - activities, - maxConcurrentWorkflowTaskExecutions: 50, - taskQueue: SYNC_TASK_QUEUE - }; - - const webhookWorker = { - connection, - namespace, - workflowsPath: createRequire(import.meta.url).resolve('./workflows'), - activities, - maxConcurrentWorkflowTaskExecutions: 50, - maxActivitiesPerSecond: 50, - taskQueue: WEBHOOK_TASK_QUEUE - }; - - const workers = await Promise.all([Worker.create(syncWorker), Worker.create(webhookWorker)]); - await Promise.all(workers.map((worker) => worker.run())); -} - -run().catch((err) => { - console.error(err); - process.exit(1); -}); diff --git a/packages/worker/lib/workflows.ts b/packages/worker/lib/workflows.ts deleted file mode 100644 index bb0d50c199d..00000000000 --- a/packages/worker/lib/workflows.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { proxyActivities } from '@temporalio/workflow'; -import type * as activities from './activities.js'; -import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs, WebhookArgs } from './models/Worker'; - -const DEFAULT_TIMEOUT = '24 hours'; -const MAXIMUM_ATTEMPTS = 3; - -const { reportFailure, routeSync, scheduleAndRouteSync, runAction, runWebhook } = proxyActivities({ - startToCloseTimeout: DEFAULT_TIMEOUT, - scheduleToCloseTimeout: DEFAULT_TIMEOUT, - retry: { - initialInterval: '5m', - maximumAttempts: MAXIMUM_ATTEMPTS - }, - heartbeatTimeout: '30m' -}); - -export async function initialSync(args: InitialSyncArgs): Promise { - try { - return await routeSync(args); - } catch (e: any) { - await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); - - return false; - } -} - -export async function continuousSync(args: ContinuousSyncArgs): Promise { - try { - return await scheduleAndRouteSync(args); - } catch (e: any) { - await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); - - return false; - } -} - -export async function action(args: ActionArgs): Promise { - try { - return await runAction(args); - } catch (e: any) { - await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); - - return { success: false }; - } -} - -export async function webhook(args: WebhookArgs): Promise { - try { - return await runWebhook(args); - } catch (e: any) { - await reportFailure(e, args, DEFAULT_TIMEOUT, MAXIMUM_ATTEMPTS); - - return false; - } -} diff --git a/packages/worker/nodemon.json b/packages/worker/nodemon.json deleted file mode 100644 index cc07c810f96..00000000000 --- a/packages/worker/nodemon.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "watch": ["lib", "../shared/lib", "../../.env"], - "ext": "ts,json", - "ignore": ["lib/**/*.spec.ts"], - "exec": "tsx -r dotenv/config lib/worker.ts dotenv_config_path=./../../.env" -} diff --git a/packages/worker/package.json b/packages/worker/package.json deleted file mode 100644 index 8a67a72a0ee..00000000000 --- a/packages/worker/package.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "@nangohq/nango-worker", - "version": "0.36.64", - "type": "module", - "main": "dist/worker.js", - "scripts": { - "start": "tsc && node dist/worker.js", - "dev": "nodemon", - "build": "tsc" - }, - "keywords": [], - "repository": { - "type": "git", - "url": "https://github.com/NangoHQ/nango", - "directory": "packages/worker" - }, - "dependencies": { - "@nangohq/shared": "0.36.64", - "@octokit/plugin-retry": "^6.0.0", - "@octokit/plugin-throttling": "^7.0.0", - "@octokit/rest": "^20.0.1", - "@temporalio/activity": "^1.5.2", - "@temporalio/client": "^1.5.2", - "@temporalio/worker": "^1.5.2", - "@temporalio/workflow": "^1.5.2", - "@types/fs-extra": "^11.0.1", - "dotenv": "^16.0.3", - "fs-extra": "^11.1.1", - "js-yaml": "^4.1.0", - "knex": "^2.4.2", - "md5": "^2.3.0", - "nanoid": "3.x", - "uuid": "^9.0.0", - "vm2": "^3.9.19" - }, - "devDependencies": { - "@tsconfig/node16": "^1.0.0", - "@types/md5": "^2.3.2", - "@types/node": "^18.7.6", - "@typescript-eslint/eslint-plugin": "^5.0.0", - "@typescript-eslint/parser": "^5.0.0", - "eslint": "^7.32.0", - "eslint-config-prettier": "^8.3.0", - "eslint-plugin-deprecation": "^1.2.1", - "nodemon": "^3.0.1", - "typescript": "^4.4.2" - } -} diff --git a/packages/worker/tsconfig.json b/packages/worker/tsconfig.json deleted file mode 100644 index fceecc91c2c..00000000000 --- a/packages/worker/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "rootDir": "lib", - "outDir": "dist" - }, - "references": [{ "path": "../shared" }], - "include": ["lib/**/*"] -} diff --git a/scripts/beta-release.bash b/scripts/beta-release.bash deleted file mode 100755 index 9cf8fdb099c..00000000000 --- a/scripts/beta-release.bash +++ /dev/null @@ -1,108 +0,0 @@ -#!/bin/bash - -# Steps involved -# 1. If any files changed in the shared package, build the webapp -# - Use the updated shared version in server, worker, cli, node-client -# 2. Build all the packages -# 3. Run a docker publish of the worker and the server -# 4. Bump the docker image version of both the worker and the server -# 5. Bump the cli version in the package.json and publish it -# -if [ $# -lt 2 ] -then - echo "Usage: ./beta-release.bash [server_version] [worker_version]" - exit 1 -fi - -function update_shared_dep() { - PACKAGE_JSON=$1 - NEW_VERSION=$2 - - # Update the version in the JSON file - jq --arg NEW_VERSION "$NEW_VERSION" '.dependencies["@nangohq/shared"] = $NEW_VERSION' --indent 4 $PACKAGE_JSON | sponge $PACKAGE_JSON -} - -function update_package_json_version() { - PACKAGE_JSON=$1 - - VERSION=$(jq -r '.version' $PACKAGE_JSON) - - # Remove '-beta' from the version - BASE_VERSION=${VERSION%-beta} - - # Split the version into its parts - IFS='.' read -ra VERSION_PARTS <<< "$BASE_VERSION" - - # Get the last part of the version - LAST_PART=${VERSION_PARTS[${#VERSION_PARTS[@]}-1]} - - # Increment the last part by 1 - NEW_LAST_PART=$((LAST_PART + 1)) - - # Replace the last part in the version - NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$NEW_LAST_PART" - - # Add '-beta' back to the version - NEW_VERSION="${NEW_VERSION}-beta" - - # Update the version in the JSON file - jq --arg NEW_VERSION "$NEW_VERSION" '.version = $NEW_VERSION' --indent 4 $PACKAGE_JSON | sponge $PACKAGE_JSON - - echo "$PACKAGE_JSON version bumped successfully!" -} - - -# STEP 1 -git update-index -q --refresh - -# Check for modifications in the shared directory and if there are publish a new version and bump the version in the other packages -if git diff --quiet -- ./packages/shared || git diff --cached --quiet -- ./packages/shared; then - SHARED_PACKAGE_JSON="packages/shared/package.json" - update_package_json_version $SHARED_PACKAGE_JSON - update_shared_dep "packages/server/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) - update_shared_dep "packages/worker/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) - update_shared_dep "packages/cli/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) - update_shared_dep "packages/node-client/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) - - rm -rf packages/shared/dist - npm run ts-build - cd ./packages/shared && npm publish --tag beta --access public && cd ../../ - NODE_CLIENT_PACKAGE_JSON="packages/node-client/package.json" - update_package_json_version $NODE_CLIENT_PACKAGE_JSON - npm i - npm run ts-build - cd ./packages/node-client && npm publish --tag beta --access public && cd ../../ -fi - -SERVER_VERSION=$1 -WORKER_VERSION=$2 -DOCKER_COMPOSE_FILE="packages/cli/docker/docker-compose.yaml" - -# STEP 2 -npm i -npm run ts-build -cd ./packages/webapp && npm run build && cd ../../ - -# STEP 3 -- run as background because it takes a while and it is non blocking -./scripts/docker-publish.bash nango-server $SERVER_VERSION-beta false hosted & -./scripts/docker-publish.bash nango-worker $WORKER_VERSION-beta true hosted & - -SERVER_IMAGE="nangohq/nango-server" -WORKER_IMAGE="nangohq/nango-worker" - -# STEP 4 -# Replace the version for nango-server -sed -i "" "s|${SERVER_IMAGE}:[^ ]*|${SERVER_IMAGE}:${SERVER_VERSION}-beta|g" $DOCKER_COMPOSE_FILE - -# Replace the version for nango-worker -sed -i "" "s|${WORKER_IMAGE}:[^ ]*|${WORKER_IMAGE}:${WORKER_VERSION}-beta|g" $DOCKER_COMPOSE_FILE - -wait - -echo "nango-server and nango-worker published successfully and docker-compose in the cli was updated" - -# STEP 5 -CLI_PACKAGE_JSON="packages/cli/package.json" -update_package_json_version $CLI_PACKAGE_JSON - -cd ./packages/cli && npm publish --tag beta --access public && cd ../../ diff --git a/scripts/release.bash b/scripts/release.bash index f074c472a72..7f40aec1a0c 100755 --- a/scripts/release.bash +++ b/scripts/release.bash @@ -2,7 +2,7 @@ if [ $# -lt 2 ] then - echo "Usage: ./release.bash [server_and_worker_version] [prod|staging|hosted] [optional_specific_version]" + echo "Usage: ./release.bash [server_version] [prod|staging|hosted] [optional_specific_version]" exit 1 fi @@ -57,7 +57,7 @@ function update_package_json_version() { # 2) Shared depends on the node-client so update the shared dependency on node-client. # npm install to update that dependency. # 3) Bump the shared package version, install build and publish it -# 4) server, worker, and cli all depend on the shared package. Update their shared +# 4) server and cli all depend on the shared package. Update their shared # dependency. # 5) The frontend has no dependencies so bump that version and publish it # 6) The webapp depends on the frontend so update that dependency, install, and build it. @@ -65,8 +65,7 @@ function update_package_json_version() { # Docker publish flow # 1) Build the nango-server cloud version (staging or prod) and push it to dockerhub # 2) Build the nango-server hosted version and push it to dockerhub -# 3) Build the nango-worker hosted version and push it to dockerhub -# 4) Update the docker-compose.yaml in the cli to use the new server and worker versions +# 4) Update the docker-compose.yaml in the cli to use the new server version # 5) Bump the cli package.json, copy the NangoSync type declaration since that is used # for integration scripts. Publish the package and then npm install at the root # to ensure all dependencies are updated @@ -88,7 +87,7 @@ cd ./packages/node-client && npm run build && npm publish --access public && cd # update the shared package that depends on the node client update_node_dep "packages/shared/package.json" $(jq -r '.version' $NODE_CLIENT_PACKAGE_JSON) -# Update the shared package and then bump the cli, server and worker versions that depend on it +# Update the shared package and then bump the cli and server versions that depend on it SHARED_PACKAGE_JSON="packages/shared/package.json" update_package_json_version $SHARED_PACKAGE_JSON $3 @@ -97,7 +96,6 @@ npm run ts-build cd ./packages/shared && npm publish --access public && cd ../../ update_shared_dep "packages/server/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) -update_shared_dep "packages/worker/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) update_shared_dep "packages/cli/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) update_shared_dep "packages/jobs/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) update_shared_dep "packages/runner/package.json" $(jq -r '.version' $SHARED_PACKAGE_JSON) @@ -116,32 +114,26 @@ npm i npm run ts-build cd ./packages/webapp && npm run build && cd ../../ -SERVER_WORKER_VERSION=$1 +SERVER_VERSION=$1 ENV=$2 DOCKER_COMPOSE_FILE="packages/cli/docker/docker-compose.yaml" -WORKER_PACKAGE_JSON="packages/worker/package.json" SERVER_PACKAGE_JSON="packages/server/package.json" update_package_json_version $SERVER_PACKAGE_JSON $3 -update_package_json_version $WORKER_PACKAGE_JSON $3 # End npm package publish flow (Minus the CLI which is published last since that depends on docker versions) rm -rf ./packages/webapp/build/fonts -./scripts/docker-publish.bash nango-server $SERVER_WORKER_VERSION true $2 +./scripts/docker-publish.bash nango-server $SERVER_VERSION true $2 rm -rf ./packages/webapp/build/fonts -./scripts/docker-publish.bash nango-server $SERVER_WORKER_VERSION true hosted -rm -rf ./packages/webapp/build/fonts -./scripts/docker-publish.bash nango-worker $SERVER_WORKER_VERSION true hosted +./scripts/docker-publish.bash nango-server $SERVER_VERSION true hosted SERVER_IMAGE="nangohq/nango-server" -WORKER_IMAGE="nangohq/nango-worker" -sed -i "" "s|${SERVER_IMAGE}:[^ ]*|${SERVER_IMAGE}:${SERVER_WORKER_VERSION}|g" $DOCKER_COMPOSE_FILE -sed -i "" "s|${WORKER_IMAGE}:[^ ]*|${WORKER_IMAGE}:${SERVER_WORKER_VERSION}|g" $DOCKER_COMPOSE_FILE +sed -i "" "s|${SERVER_IMAGE}:[^ ]*|${SERVER_IMAGE}:${SERVER_VERSION}|g" $DOCKER_COMPOSE_FILE -echo "nango-server and nango-worker published successfully and docker-compose in the cli was updated" +echo "nango-server was published successfully and docker-compose in the cli was updated" CLI_PACKAGE_JSON="packages/cli/package.json" update_package_json_version $CLI_PACKAGE_JSON $3 diff --git a/tsconfig.build.json b/tsconfig.build.json index bc6fb6c92e3..b53a3f5ee69 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -3,7 +3,6 @@ "references": [ { "path": "packages/node-client" }, { "path": "packages/shared" }, - { "path": "packages/worker" }, { "path": "packages/cli" }, { "path": "packages/server" }, { "path": "packages/frontend" }, From b102a9d7b4407eeef397f2e357b54f78ec1ae926 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 20 Dec 2023 14:58:34 -0500 Subject: [PATCH 37/85] [NAN-164] Enterprise prep (#1449) * [nan-164] add enterprise logic * [nan-164] add enterprise * [nan-164] build and publish enterprise for server * [nan-164] improve spacing --- packages/jobs/lib/temporal.ts | 19 +++++++------- .../server/lib/controllers/user.controller.ts | 4 +-- packages/server/lib/server.ts | 26 +++++++++++++------ .../lib/services/nango-config.service.ts | 15 ----------- packages/shared/lib/utils/utils.ts | 4 +++ packages/webapp/.env.enterprise | 2 ++ packages/webapp/package.json | 1 + packages/webapp/src/App.tsx | 4 +-- .../webapp/src/components/PrivateRoute.tsx | 4 +-- packages/webapp/src/components/TopNavBar.tsx | 4 +-- packages/webapp/src/env.d.ts | 2 +- packages/webapp/src/pages/Syncs.tsx | 6 ++--- packages/webapp/src/utils/utils.tsx | 4 +++ scripts/docker-publish.bash | 4 +++ scripts/release.bash | 4 ++- 15 files changed, 58 insertions(+), 45 deletions(-) create mode 100644 packages/webapp/.env.enterprise diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts index d7a06185d53..5227eaaac73 100644 --- a/packages/jobs/lib/temporal.ts +++ b/packages/jobs/lib/temporal.ts @@ -3,7 +3,7 @@ import fs from 'fs-extra'; import * as dotenv from 'dotenv'; import { createRequire } from 'module'; import * as activities from './activities.js'; -import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd } from '@nangohq/shared'; +import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd, isEnterprise } from '@nangohq/shared'; export class Temporal { namespace: string; @@ -24,21 +24,22 @@ export class Temporal { let crt: Buffer | null = null; let key: Buffer | null = null; - if (isProd()) { + if (isProd() || isEnterprise()) { crt = await fs.readFile(`/etc/secrets/${this.namespace}.crt`); key = await fs.readFile(`/etc/secrets/${this.namespace}.key`); } const connection = await NativeConnection.connect({ address: process.env['TEMPORAL_ADDRESS'] || 'localhost:7233', - tls: !isProd() - ? false - : { - clientCertPair: { - crt: crt as Buffer, - key: key as Buffer + tls: + !isProd() && !isEnterprise() + ? false + : { + clientCertPair: { + crt: crt as Buffer, + key: key as Buffer + } } - } }); const syncWorker = { diff --git a/packages/server/lib/controllers/user.controller.ts b/packages/server/lib/controllers/user.controller.ts index 21a7679bdcc..6e14c708bf6 100644 --- a/packages/server/lib/controllers/user.controller.ts +++ b/packages/server/lib/controllers/user.controller.ts @@ -1,7 +1,7 @@ import { getUserAccountAndEnvironmentFromSession } from '../utils/utils.js'; import type { Request, Response, NextFunction } from 'express'; import EmailClient from '../clients/email.client.js'; -import { errorManager, userService, getBaseUrl, isCloud } from '@nangohq/shared'; +import { errorManager, userService, getBaseUrl, isCloud, isEnterprise } from '@nangohq/shared'; class UserController { async getUser(req: Request, res: Response, next: NextFunction) { @@ -99,7 +99,7 @@ class UserController { if (!invited) { throw new Error('Failed to invite user.'); } - if (isCloud()) { + if (isCloud() || isEnterprise()) { const emailClient = EmailClient.getInstance(); emailClient.send( invited.email, diff --git a/packages/server/lib/server.ts b/packages/server/lib/server.ts index fdc9e04111e..589d20498bb 100644 --- a/packages/server/lib/server.ts +++ b/packages/server/lib/server.ts @@ -34,7 +34,16 @@ import environmentController from './controllers/environment.controller.js'; import accountController from './controllers/account.controller.js'; import type { Response, Request } from 'express'; import Logger from './utils/logger.js'; -import { getGlobalOAuthCallbackUrl, environmentService, getPort, isCloud, isBasicAuthEnabled, errorManager, getWebsocketsPath } from '@nangohq/shared'; +import { + getGlobalOAuthCallbackUrl, + environmentService, + getPort, + isCloud, + isEnterprise, + isBasicAuthEnabled, + errorManager, + getWebsocketsPath +} from '@nangohq/shared'; import oAuthSessionService from './services/oauth-session.service.js'; import { deleteOldActivityLogs } from './jobs/index.js'; import migrate from './utils/migrate.js'; @@ -47,11 +56,12 @@ const app = express(); AuthClient.setup(app); const apiAuth = authMiddleware.secretKeyAuth.bind(authMiddleware); const apiPublicAuth = authMiddleware.publicKeyAuth.bind(authMiddleware); -const webAuth = isCloud() - ? [passport.authenticate('session'), authMiddleware.sessionAuth.bind(authMiddleware)] - : isBasicAuthEnabled() - ? [passport.authenticate('basic', { session: false }), authMiddleware.basicAuth.bind(authMiddleware)] - : [authMiddleware.noAuth.bind(authMiddleware)]; +const webAuth = + isCloud() || isEnterprise() + ? [passport.authenticate('session'), authMiddleware.sessionAuth.bind(authMiddleware)] + : isBasicAuthEnabled() + ? [passport.authenticate('basic', { session: false }), authMiddleware.basicAuth.bind(authMiddleware)] + : [authMiddleware.noAuth.bind(authMiddleware)]; app.use(express.json({ limit: '75mb' })); app.use(cors()); @@ -117,7 +127,7 @@ app.route('/admin/flow/deploy/pre-built').post(apiAuth, flowController.adminDepl app.route('/proxy/*').all(apiAuth, proxyController.routeCall.bind(proxyController)); // Webapp routes (no auth). -if (isCloud()) { +if (isCloud() || isEnterprise()) { app.route('/api/v1/signup').post(authController.signup.bind(authController)); app.route('/api/v1/signup/invite').get(authController.invitation.bind(authController)); app.route('/api/v1/logout').post(authController.logout.bind(authController)); @@ -183,7 +193,7 @@ app.route('/api/v1/onboarding/:id').put(webAuth, onboardingController.updateStat app.route('/api/v1/onboarding/sync-status').get(webAuth, onboardingController.checkSyncCompletion.bind(onboardingController)); // Hosted signin -if (!isCloud()) { +if (!isCloud() && !isEnterprise()) { app.route('/api/v1/basic').get(webAuth, (_: Request, res: Response) => { res.status(200).send(); }); diff --git a/packages/shared/lib/services/nango-config.service.ts b/packages/shared/lib/services/nango-config.service.ts index 14511bc0f4e..cf3d540b3f7 100644 --- a/packages/shared/lib/services/nango-config.service.ts +++ b/packages/shared/lib/services/nango-config.service.ts @@ -16,7 +16,6 @@ import type { NangoSyncEndpoint, NangoIntegrationDataV2 } from '../models/NangoConfig.js'; -import { isCloud } from '../utils/utils.js'; import type { HTTP_VERB, ServiceResponse } from '../models/Generic.js'; import { SyncType, SyncConfigType } from '../models/Sync.js'; import { NangoError } from '../utils/error.js'; @@ -87,20 +86,6 @@ export function loadStandardConfig(configData: NangoConfig, showMessages = false } } -export function getRootDir(optionalLoadLocation?: string) { - if (isCloud()) { - return './'; - } - - if (optionalLoadLocation) { - return optionalLoadLocation; - } else if (process.env['NANGO_INTEGRATIONS_FULL_PATH']) { - return `${process.env['NANGO_INTEGRATIONS_FULL_PATH']}/dist`; - } else { - return path.resolve(__dirname, '../nango-integrations/dist'); - } -} - function getFieldsForModel(modelName: string, config: NangoConfig): { name: string; type: string }[] | null { const modelFields = []; diff --git a/packages/shared/lib/utils/utils.ts b/packages/shared/lib/utils/utils.ts index 398dd5963fa..05d077710cf 100644 --- a/packages/shared/lib/utils/utils.ts +++ b/packages/shared/lib/utils/utils.ts @@ -45,6 +45,10 @@ export function isCloud() { return process.env['NANGO_CLOUD']?.toLowerCase() === 'true'; } +export function isEnterprise() { + return process.env['NANGO_ENTERPRISE']?.toLowerCase() === 'true'; +} + export function isStaging() { return process.env['NODE_ENV'] === NodeEnv.Staging; } diff --git a/packages/webapp/.env.enterprise b/packages/webapp/.env.enterprise new file mode 100644 index 00000000000..a6d9e96c690 --- /dev/null +++ b/packages/webapp/.env.enterprise @@ -0,0 +1,2 @@ +REACT_APP_ENV=enterprise + diff --git a/packages/webapp/package.json b/packages/webapp/package.json index ecd9fb0979c..97032d2ae7d 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -42,6 +42,7 @@ "start:staging": "env-cmd -f .env.staging npm run start", "start:prod": "env-cmd -f .env.prod npm run start", "build:hosted": "env-cmd -f .env.hosted npm run build", + "build:enterprise": "env-cmd -f .env.enterprise npm run build", "build:staging": "env-cmd -f .env.staging npm run build", "build:prod": "env-cmd -f .env.prod npm run build" }, diff --git a/packages/webapp/src/App.tsx b/packages/webapp/src/App.tsx index 48701290b2e..6e3031ca6f3 100644 --- a/packages/webapp/src/App.tsx +++ b/packages/webapp/src/App.tsx @@ -32,7 +32,7 @@ import AccountSettings from './pages/AccountSettings'; import UserSettings from './pages/UserSettings'; import { ToastContainer } from 'react-toastify'; import 'react-toastify/dist/ReactToastify.css'; -import { isCloud } from './utils/utils'; +import { isCloud, isEnterprise } from './utils/utils'; import { useStore } from './store'; Sentry.init({ @@ -113,7 +113,7 @@ const App = () => { }> } /> - {isCloud() && ( + {(isCloud() || isEnterprise()) && ( <> } /> } /> diff --git a/packages/webapp/src/components/PrivateRoute.tsx b/packages/webapp/src/components/PrivateRoute.tsx index c9f3962ab11..e878ef9ee7c 100644 --- a/packages/webapp/src/components/PrivateRoute.tsx +++ b/packages/webapp/src/components/PrivateRoute.tsx @@ -1,9 +1,9 @@ import { Outlet, Navigate } from 'react-router-dom'; -import { isCloud } from '../utils/utils'; +import { isCloud, isEnterprise } from '../utils/utils'; import { isSignedIn } from '../utils/user'; const PrivateRoute = (_: any) => { - return <>{!isCloud() || isSignedIn() ? : }; + return <>{(!isCloud() && !isEnterprise()) || isSignedIn() ? : }; }; export default PrivateRoute; diff --git a/packages/webapp/src/components/TopNavBar.tsx b/packages/webapp/src/components/TopNavBar.tsx index 1f867a3c375..904e139f7d6 100644 --- a/packages/webapp/src/components/TopNavBar.tsx +++ b/packages/webapp/src/components/TopNavBar.tsx @@ -1,5 +1,5 @@ import { Book, Slack, Github } from '@geist-ui/icons'; -import { isCloud } from '../utils/utils'; +import { isCloud, isEnterprise } from '../utils/utils'; import { useSignout } from '../utils/user'; export default function NavBar() { @@ -43,7 +43,7 @@ export default function NavBar() {

Github

- {isCloud() && ( + {(isCloud() || isEnterprise()) && (