From 369c423e5b1dfc777270e5807300e744a211a807 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 27 Jun 2023 10:04:37 -0400 Subject: [PATCH 01/45] runtime: allow globals to be injected --- packages/runtime/src/execute/context.ts | 5 +++- packages/runtime/src/runtime.ts | 5 ++++ packages/runtime/test/runtime.test.ts | 12 ++++++++++ pnpm-lock.yaml | 31 +++++++------------------ 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/packages/runtime/src/execute/context.ts b/packages/runtime/src/execute/context.ts index 10bfeb624..585567199 100644 --- a/packages/runtime/src/execute/context.ts +++ b/packages/runtime/src/execute/context.ts @@ -15,11 +15,14 @@ const freezeAll = ( // Build a safe and helpful execution context // This will be shared by all jobs -export default (state: State, options: Pick) => { +export default (state: State, options: Pick) => { const logger = options.jobLogger ?? console; + const globals = options.globals || {}; const context = vm.createContext( freezeAll( { + ...globals, + // Note that these globals will be overridden console: logger, clearInterval, clearTimeout, diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index d00010901..5830e7e33 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -27,6 +27,11 @@ export type Options = { forceSandbox?: boolean; linker?: LinkerOptions; + + // inject stuff into the environment + // aka globals + // Used by unit tests. any security concerns? + globals?: any; }; const defaultState = { data: {}, configuration: {} }; diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 73fd6477d..6db890e09 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -269,3 +269,15 @@ test('stuff written to state before an error is preserved', async (t) => { t.is(result.x, 1); }); + +test('inject globals', async (t) => { + const expression = 'export default [(s) => Object.assign(s, { data: { x } })]'; + + const result: any = await run(expression, {}, { + globals: { + x: 90210 + } + }); + t.is(result.data.x, 90210); +}); + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b3e0e7e6..c9696925c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,14 +59,6 @@ importers: tslib: 2.4.0 typescript: 4.8.3 - integration-tests/cli/repo: - specifiers: - '@openfn/language-common_1.7.7': npm:@openfn/language-common@^1.7.7 - is-array_1.0.1: npm:is-array@^1.0.1 - dependencies: - '@openfn/language-common_1.7.7': /@openfn/language-common/1.7.7 - is-array_1.0.1: /is-array/1.0.1 - packages/cli: specifiers: '@openfn/compiler': workspace:* @@ -618,17 +610,6 @@ packages: - debug dev: true - /@openfn/language-common/1.7.7: - resolution: {integrity: sha512-GSoAbo6oL0b8jHufhLKvIzHJ271aE2AKv/ibeuiWU3CqN1gRmaHArlA/omlCs/rsfcieSp2VWAvWeGuFY8buZw==} - dependencies: - axios: 1.1.3 - date-fns: 2.29.3 - jsonpath-plus: 4.0.0 - lodash: 4.17.21 - transitivePeerDependencies: - - debug - dev: false - /@openfn/language-common/2.0.0-rc3: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] @@ -1072,6 +1053,7 @@ packages: /asynckit/0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true /atob/2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -1145,6 +1127,7 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: true /b4a/1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -1540,6 +1523,7 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 + dev: true /commander/4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -1810,6 +1794,7 @@ packages: /delayed-stream/1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dev: true /delegates/1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -2849,6 +2834,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: true /for-in/1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -2862,6 +2848,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: true /fragment-cache/0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -3264,10 +3251,6 @@ packages: kind-of: 6.0.3 dev: true - /is-array/1.0.1: - resolution: {integrity: sha512-gxiZ+y/u67AzpeFmAmo4CbtME/bs7J2C++su5zQzvQyaxUqVzkh69DI+jN+KZuSO6JaH6TIIU6M6LhqxMjxEpw==} - dev: false - /is-arrayish/0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} dev: true @@ -3584,6 +3567,7 @@ packages: /jsonpath-plus/4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} + dev: true /keygrip/1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} @@ -4533,6 +4517,7 @@ packages: /proxy-from-env/1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true /proxy-middleware/0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} From 6002a0f58a45315d0fc88daab0baf521e212d110 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 27 Jun 2023 10:13:30 -0400 Subject: [PATCH 02/45] runtime: extra test on global injection --- packages/runtime/test/runtime.test.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 6db890e09..0ce9226c3 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -281,3 +281,28 @@ test('inject globals', async (t) => { t.is(result.data.x, 90210); }); +test('injected globals can\'t override special functions', async (t) => { + const panic = () => { throw new Error('illegal override') } + + const globals = { + console: panic, + clearInterval: panic, + clearTimeout: panic, + parseFloat: panic, + parseInt: panic, + setInterval: panic, + setTimeout: panic, + } + const expression = `export default [(s) => { + parseFloat(); + parseInt(); + const i = setInterval(() => {}, 1000); + clearInterval(i); + const t = setTimeout(() => {}, 1000); + clearTimeout(t); + return s; + }]`; + + const result: any = await run(expression, {}, { globals }); + t.falsy(result.errors); +}); From a520e7b7aa7ed65119902efd877300ee62687f32 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 27 Jun 2023 10:46:41 -0400 Subject: [PATCH 03/45] runtime: use global execute from injection if it exists --- packages/runtime/src/execute/expression.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts index 7ac7a49e1..d03d0a978 100644 --- a/packages/runtime/src/execute/expression.ts +++ b/packages/runtime/src/execute/expression.ts @@ -23,7 +23,7 @@ export default ( const { operations, execute } = await prepareJob(expression, context, opts); // Create the main reducer function - const reducer = (execute || defaultExecute)( + const reducer = (execute || opts.globals?.execute || defaultExecute)( ...operations.map((op, idx) => wrapOperation(op, logger, `${idx + 1}`, opts.immutableState) ) From fad04a9a5d2371991ae2cf59dd01789bc17fe63b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 12:39:29 +0000 Subject: [PATCH 04/45] lightning-mock: refactor tests --- .../attempt.test.ts} | 138 +----------------- .../test/channels/claim.test.ts | 109 ++++++++++++++ packages/lightning-mock/test/server.test.ts | 110 ++++++++++++++ 3 files changed, 223 insertions(+), 134 deletions(-) rename packages/lightning-mock/test/{lightning.test.ts => channels/attempt.test.ts} (63%) create mode 100644 packages/lightning-mock/test/channels/claim.test.ts create mode 100644 packages/lightning-mock/test/server.test.ts diff --git a/packages/lightning-mock/test/lightning.test.ts b/packages/lightning-mock/test/channels/attempt.test.ts similarity index 63% rename from packages/lightning-mock/test/lightning.test.ts rename to packages/lightning-mock/test/channels/attempt.test.ts index 6a40b21eb..2e338f72d 100644 --- a/packages/lightning-mock/test/lightning.test.ts +++ b/packages/lightning-mock/test/channels/attempt.test.ts @@ -1,10 +1,10 @@ import test from 'ava'; -import createLightningServer from '../src/server'; +import createLightningServer from '../../src/server'; import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; -import { attempts, credentials, dataclips } from './data'; +import { attempts, credentials, dataclips } from '../data'; import { ATTEMPT_COMPLETE, AttemptCompletePayload, @@ -13,8 +13,8 @@ import { GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, -} from '../src/events'; -import type { Attempt } from '../src/types'; +} from '../../src/events'; +import type { Attempt } from '../../src/types'; import { JSONLog } from '@openfn/logger'; const endpoint = 'ws://localhost:7777/worker'; @@ -68,134 +68,6 @@ const join = (channelName: string, params: any = {}): Promise => }); }); -// Test some dev hooks -// enqueue attempt should add id to the queue and register the state, credentials and body -test.serial('should setup an attempt at /POST /attempt', async (t) => { - const state = server.getState(); - - t.is(Object.keys(state.credentials).length, 0); - t.is(Object.keys(state.attempts).length, 0); - t.is(Object.keys(state.attempts).length, 0); - - const attempt: Attempt = { - id: 'a', - triggers: [], - jobs: [ - { - body: 'abc', - dataclip: {}, // not sure how this will work on the attempt yet - credential: { - // this will be converted into a string for lazy loading - user: 'john', - password: 'rambo', - }, - }, - ], - edges: [], - }; - - await fetch('http://localhost:7777/attempt', { - method: 'POST', - body: JSON.stringify(attempt), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); - - const newState = server.getState(); - t.is(Object.keys(newState.attempts).length, 1); - const a = server.getAttempt('a'); - t.truthy(a); - t.is(server.getQueueLength(), 1); - - t.is(Object.keys(newState.credentials).length, 1); - - const job = a.jobs[0]; - t.assert(typeof job.credential === 'string'); - const c = server.getCredential(job.credential); - t.is(c.user, 'john'); -}); - -test.serial('provide a phoenix websocket at /worker', (t) => { - // client should be connected before this test runs - t.is(client.connectionState(), 'open'); -}); - -test.serial('reject ws connections without a token', (t) => { - return new Promise((done) => { - // client should be connected before this test runs - const socket = new Socket(endpoint, { transport: WebSocket }); - socket.onClose(() => { - t.pass(); - done(); - }); - socket.connect(); - }); -}); - -test.serial('respond to channel join requests', (t) => { - return new Promise(async (done, reject) => { - const channel = client.channel('x', {}); - - channel.join().receive('ok', (res) => { - t.is(res, 'ok'); - done(); - }); - }); -}); - -// TODO: only allow authorised workers to join workers -// TODO: only allow authorised attemtps to join an attempt channel - -test.serial( - 'claim attempt: reply for zero items if queue is empty', - (t) => - new Promise(async (done) => { - t.is(server.getQueueLength(), 0); - - const channel = await join('worker:queue'); - - // response is an array of attempt ids - channel.push(CLAIM).receive('ok', (response) => { - const { attempts } = response; - t.assert(Array.isArray(attempts)); - t.is(attempts.length, 0); - - t.is(server.getQueueLength(), 0); - done(); - }); - }) -); - -test.serial( - "claim attempt: reply with an attempt id if there's an attempt in the queue", - (t) => - new Promise(async (done) => { - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 1); - - const channel = await join('worker:queue'); - - // response is an array of attempt ids - channel.push(CLAIM).receive('ok', (response) => { - const { attempts } = response; - t.truthy(attempts); - t.is(attempts.length, 1); - t.deepEqual(attempts[0], { id: 'attempt-1', token: 'x.y.z' }); - - // ensure the server state has changed - t.is(server.getQueueLength(), 0); - done(); - }); - }) -); - -// TODO is it even worth doing this? Easier for a socket to pull one at a time? -// It would also ensure better distribution if 10 workers ask at the same time, they'll get -// one each then come back for more -test.todo('claim attempt: reply with multiple attempt ids'); - test.serial('create a channel for an attempt', async (t) => { server.startAttempt('wibble'); await join('attempt:wibble', { token: 'a.b.c' }); @@ -209,8 +81,6 @@ test.serial('do not allow to join a channel without a token', async (t) => { }); }); -test.todo('do not allow to join a channel without a valid token'); - test.serial('reject channels for attempts that are not started', async (t) => { await t.throwsAsync(() => join('attempt:xyz'), { message: 'invalid_attempt_id', diff --git a/packages/lightning-mock/test/channels/claim.test.ts b/packages/lightning-mock/test/channels/claim.test.ts new file mode 100644 index 000000000..be929e3db --- /dev/null +++ b/packages/lightning-mock/test/channels/claim.test.ts @@ -0,0 +1,109 @@ +import test from 'ava'; +import createLightningServer from '../../src/server'; + +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; + +import { attempts } from '../data'; +import { CLAIM } from '../../src/events'; + +const port = 4444; + +const endpoint = `ws://localhost:${port}/worker`; + +type Channel = any; + +let server; +let client; + +// Set up a lightning server and a phoenix socket client before each test +test.before( + () => + new Promise((done) => { + server = createLightningServer({ port: 4444 }); + + // Note that we need a token to connect, but the mock here + // doesn't (yet) do any validation on that token + client = new Socket(endpoint, { + params: { token: 'x.y.z' }, + timeout: 1000 * 120, + transport: WebSocket, + }); + client.onOpen(done); + client.connect(); + }) +); + +test.afterEach(() => { + server.reset(); +}); + +test.after(() => { + server.destroy(); +}); + +const attempt1 = attempts['attempt-1']; + +const join = (channelName: string, params: any = {}): Promise => + new Promise((done, reject) => { + const channel = client.channel(channelName, params); + channel + .join() + .receive('ok', () => { + done(channel); + }) + .receive('error', (err) => { + // err will be the response message on the payload (ie, invalid_token, invalid_attempt_id etc) + reject(new Error(err)); + }); + }); + +test.serial( + 'claim attempt: reply for zero items if queue is empty', + (t) => + new Promise(async (done) => { + t.is(server.getQueueLength(), 0); + + const channel = await join('worker:queue'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + const { attempts } = response; + t.assert(Array.isArray(attempts)); + t.is(attempts.length, 0); + + t.is(server.getQueueLength(), 0); + done(); + }); + }) +); + +test.serial( + "claim attempt: reply with an attempt id if there's an attempt in the queue", + (t) => + new Promise(async (done) => { + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 1); + + const channel = await join('worker:queue'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + const { attempts } = response; + t.truthy(attempts); + t.is(attempts.length, 1); + t.deepEqual(attempts[0], { id: 'attempt-1', token: 'x.y.z' }); + + // ensure the server state has changed + t.is(server.getQueueLength(), 0); + done(); + }); + }) +); + +// TODO is it even worth doing this? Easier for a socket to pull one at a time? +// It would also ensure better distribution if 10 workers ask at the same time, they'll get +// one each then come back for more +test.todo('claim attempt: reply with multiple attempt ids'); + +test.todo('token auth'); diff --git a/packages/lightning-mock/test/server.test.ts b/packages/lightning-mock/test/server.test.ts new file mode 100644 index 000000000..116571833 --- /dev/null +++ b/packages/lightning-mock/test/server.test.ts @@ -0,0 +1,110 @@ +// Tests of the lightning websever +import test from 'ava'; +import createLightningServer from '../src/server'; + +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; + +import type { Attempt } from '../src/types'; + +let server; +let client; + +const port = 3333; + +const endpoint = `ws://localhost:${port}/worker`; + +// Set up a lightning server and a phoenix socket client before each test +test.before( + () => + new Promise((done) => { + server = createLightningServer({ port }); + + // Note that we need a token to connect, but the mock here + // doesn't (yet) do any validation on that token + client = new Socket(endpoint, { + params: { token: 'x.y.z' }, + timeout: 1000 * 120, + transport: WebSocket, + }); + client.onOpen(done); + client.connect(); + }) +); + +test.serial('should setup an attempt at /POST /attempt', async (t) => { + const state = server.getState(); + + t.is(Object.keys(state.credentials).length, 0); + t.is(Object.keys(state.attempts).length, 0); + t.is(Object.keys(state.attempts).length, 0); + + const attempt: Attempt = { + id: 'a', + dataclip_id: 'a', + starting_node_id: 'j', + triggers: [], + jobs: [ + { + id: 'j', + body: 'abc', + credential: { + // this will be converted into a string for lazy loading + user: 'john', + password: 'rambo', + }, + }, + ], + edges: [], + }; + + await fetch(`http://localhost:${port}/attempt`, { + method: 'POST', + body: JSON.stringify(attempt), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + const newState = server.getState(); + t.is(Object.keys(newState.attempts).length, 1); + const a = server.getAttempt('a'); + t.truthy(a); + t.is(server.getQueueLength(), 1); + + t.is(Object.keys(newState.credentials).length, 1); + + const job = a.jobs[0]; + t.assert(typeof job.credential === 'string'); + const c = server.getCredential(job.credential); + t.is(c.user, 'john'); +}); + +test.serial('provide a phoenix websocket at /worker', (t) => { + // client should be connected before this test runs + t.is(client.connectionState(), 'open'); +}); + +test.serial('reject ws connections without a token', (t) => { + return new Promise((done) => { + // client should be connected before this test runs + const socket = new Socket(endpoint, { transport: WebSocket }); + socket.onClose(() => { + t.pass(); + done(); + }); + socket.connect(); + }); +}); + +test.serial('respond to channel join requests', (t) => { + return new Promise(async (done, reject) => { + const channel = client.channel('x', {}); + + channel.join().receive('ok', (res) => { + t.is(res, 'ok'); + done(); + }); + }); +}); From fe67c203a1dcdef31c296b5fd305ef10e5c4d85d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 12:49:09 +0000 Subject: [PATCH 05/45] lightning-mock: tidy, simplify --- .../test/channels/attempt.test.ts | 28 +++---------------- .../test/channels/claim.test.ts | 25 ++--------------- packages/lightning-mock/test/server.test.ts | 21 ++------------ packages/lightning-mock/test/util.ts | 22 +++++++++++++++ 4 files changed, 30 insertions(+), 66 deletions(-) create mode 100644 packages/lightning-mock/test/util.ts diff --git a/packages/lightning-mock/test/channels/attempt.test.ts b/packages/lightning-mock/test/channels/attempt.test.ts index 2e338f72d..2fce1e11e 100644 --- a/packages/lightning-mock/test/channels/attempt.test.ts +++ b/packages/lightning-mock/test/channels/attempt.test.ts @@ -1,48 +1,28 @@ import test from 'ava'; -import createLightningServer from '../../src/server'; - -import { Socket } from 'phoenix'; -import { WebSocket } from 'ws'; +import { setup } from '../util'; import { attempts, credentials, dataclips } from '../data'; import { ATTEMPT_COMPLETE, AttemptCompletePayload, ATTEMPT_LOG, - CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, } from '../../src/events'; -import type { Attempt } from '../../src/types'; import { JSONLog } from '@openfn/logger'; -const endpoint = 'ws://localhost:7777/worker'; - const enc = new TextDecoder('utf-8'); type Channel = any; +const port = 7777; + let server; let client; // Set up a lightning server and a phoenix socket client before each test -test.before( - () => - new Promise((done) => { - server = createLightningServer({ port: 7777 }); - - // Note that we need a token to connect, but the mock here - // doesn't (yet) do any validation on that token - client = new Socket(endpoint, { - params: { token: 'x.y.z' }, - timeout: 1000 * 120, - transport: WebSocket, - }); - client.onOpen(done); - client.connect(); - }) -); +test.before(async () => ({ server, client } = await setup(port))); test.afterEach(() => { server.reset(); diff --git a/packages/lightning-mock/test/channels/claim.test.ts b/packages/lightning-mock/test/channels/claim.test.ts index be929e3db..adcd54aa1 100644 --- a/packages/lightning-mock/test/channels/claim.test.ts +++ b/packages/lightning-mock/test/channels/claim.test.ts @@ -1,38 +1,17 @@ import test from 'ava'; -import createLightningServer from '../../src/server'; - -import { Socket } from 'phoenix'; -import { WebSocket } from 'ws'; +import { setup } from '../util'; import { attempts } from '../data'; import { CLAIM } from '../../src/events'; const port = 4444; -const endpoint = `ws://localhost:${port}/worker`; - type Channel = any; let server; let client; -// Set up a lightning server and a phoenix socket client before each test -test.before( - () => - new Promise((done) => { - server = createLightningServer({ port: 4444 }); - - // Note that we need a token to connect, but the mock here - // doesn't (yet) do any validation on that token - client = new Socket(endpoint, { - params: { token: 'x.y.z' }, - timeout: 1000 * 120, - transport: WebSocket, - }); - client.onOpen(done); - client.connect(); - }) -); +test.before(async () => ({ server, client } = await setup(port))); test.afterEach(() => { server.reset(); diff --git a/packages/lightning-mock/test/server.test.ts b/packages/lightning-mock/test/server.test.ts index 116571833..b42c21061 100644 --- a/packages/lightning-mock/test/server.test.ts +++ b/packages/lightning-mock/test/server.test.ts @@ -1,10 +1,9 @@ // Tests of the lightning websever import test from 'ava'; -import createLightningServer from '../src/server'; - import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; +import { setup } from './util'; import type { Attempt } from '../src/types'; let server; @@ -14,23 +13,7 @@ const port = 3333; const endpoint = `ws://localhost:${port}/worker`; -// Set up a lightning server and a phoenix socket client before each test -test.before( - () => - new Promise((done) => { - server = createLightningServer({ port }); - - // Note that we need a token to connect, but the mock here - // doesn't (yet) do any validation on that token - client = new Socket(endpoint, { - params: { token: 'x.y.z' }, - timeout: 1000 * 120, - transport: WebSocket, - }); - client.onOpen(done); - client.connect(); - }) -); +test.before(async () => ({ server, client } = await setup(port))); test.serial('should setup an attempt at /POST /attempt', async (t) => { const state = server.getState(); diff --git a/packages/lightning-mock/test/util.ts b/packages/lightning-mock/test/util.ts new file mode 100644 index 000000000..a96d2c149 --- /dev/null +++ b/packages/lightning-mock/test/util.ts @@ -0,0 +1,22 @@ +import createLightningServer from '../src/server'; + +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; + +export const setup = (port: number) => { + return new Promise<{ server: any; client: any }>((done) => { + const server = createLightningServer({ port }); + + // Note that we need a token to connect, but the mock here + // doesn't (yet) do any validation on that token + const client = new Socket(`ws://localhost:${port}/worker`, { + params: { token: 'x.y.z' }, + timeout: 1000 * 120, + transport: WebSocket, + }); + client.onOpen(() => { + done({ server, client }); + }); + client.connect(); + }); +}; From 606c6f0184c40b460449f0ca5852f70382f17a6b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 14:33:53 +0000 Subject: [PATCH 06/45] lightning-mock: use ws-worker types A stronger source of truth --- packages/lightning-mock/package.json | 1 + packages/lightning-mock/src/api-dev.ts | 6 +- packages/lightning-mock/src/api-sockets.ts | 25 +++---- packages/lightning-mock/src/events.ts | 66 +------------------ packages/lightning-mock/src/server.ts | 2 +- .../test/events/attempt-start.test.ts | 62 +++++++++++++++++ 6 files changed, 83 insertions(+), 79 deletions(-) create mode 100644 packages/lightning-mock/test/events/attempt-start.test.ts diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 3114c42fe..c7c549b25 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -20,6 +20,7 @@ "@openfn/engine-multi": "workspace:*", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", + "@openfn/ws-worker": "workspace:^", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index 95260e5b8..9a7662eb2 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -6,10 +6,10 @@ import Koa from 'koa'; import Router from '@koa/router'; import { Logger } from '@openfn/logger'; import crypto from 'node:crypto'; +import { ATTEMPT_COMPLETE, AttemptCompletePayload } from '@openfn/ws-worker'; import { Attempt } from './types'; import { ServerState } from './server'; -import { ATTEMPT_COMPLETE, AttemptCompletePayload } from './events'; type LightningEvents = 'log' | 'attempt-complete'; @@ -91,7 +91,7 @@ const setupDevAPI = ( }) => { if (evt.attemptId === attemptId) { state.events.removeListener(ATTEMPT_COMPLETE, handler); - const result = state.dataclips[evt.payload.final_dataclip_id]; + const result = state.dataclips[evt.payload.final_dataclip_id!]; resolve(result); } }; @@ -131,7 +131,7 @@ const setupDevAPI = ( event: LightningEvents, attemptId: string, fn: (evt: any) => void, - once = true, + once = true ) => { function handler(e: any) { if (e.attemptId && e.attemptId === attemptId) { diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index b7c49d5ee..aa5d16d1c 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -9,34 +9,35 @@ import createPheonixMockSocketServer, { DevSocket, PhoenixEvent, } from './socket-server'; + import { ATTEMPT_COMPLETE, + ATTEMPT_LOG, + ATTEMPT_START, AttemptCompletePayload, AttemptCompleteReply, - ATTEMPT_LOG, AttemptLogPayload, AttemptLogReply, CLAIM, + ClaimAttempt, ClaimPayload, ClaimReply, - ClaimAttempt, GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, GetAttemptPayload, GetAttemptReply, - GET_CREDENTIAL, GetCredentialPayload, GetCredentialReply, - GET_DATACLIP, GetDataclipPayload, GetDataClipReply, RUN_COMPLETE, - RunCompletePayload, RUN_START, - RunStart, - RunStartReply, + RunCompletePayload, RunCompleteReply, - ATTEMPT_START, -} from './events'; + RunStartPayload, + RunStartReply, +} from '@openfn/ws-worker'; import type { Server } from 'http'; import { stringify } from './util'; @@ -296,7 +297,9 @@ const createSocketAPI = ( if (!state.results[attemptId]) { state.results[attemptId] = { state: null, workerId: 'mock' }; } - state.results[attemptId].state = state.dataclips[final_dataclip_id]; + if (final_dataclip_id) { + state.results[attemptId].state = state.dataclips[final_dataclip_id]; + } ws.reply({ ref, @@ -311,7 +314,7 @@ const createSocketAPI = ( function handleRunStart( state: ServerState, ws: DevSocket, - evt: PhoenixEvent + evt: PhoenixEvent ) { const { ref, join_ref, topic } = evt; if (!state.dataclips) { diff --git a/packages/lightning-mock/src/events.ts b/packages/lightning-mock/src/events.ts index f098dea44..163b5c808 100644 --- a/packages/lightning-mock/src/events.ts +++ b/packages/lightning-mock/src/events.ts @@ -1,75 +1,13 @@ -import { Attempt } from './types'; - /** * These events are copied out of ws-worker * There is a danger of them diverging */ +// TODO yeah for sure we need to remove these from here. Use the worker's types. + // new client connected export const CONNECT = 'socket:connect'; // client left or joined a channel export const CHANNEL_JOIN = 'socket:channel-join'; export const CHANNEL_LEAVE = 'socket:channel-leave'; - -export const CLAIM = 'claim'; - -export type ClaimPayload = { demand?: number }; -export type ClaimReply = { attempts: Array }; -export type ClaimAttempt = { id: string; token: string }; - -export const GET_ATTEMPT = 'fetch:attempt'; -export type GetAttemptPayload = void; // no payload -export type GetAttemptReply = Attempt; - -export const GET_CREDENTIAL = 'fetch:credential'; -export type GetCredentialPayload = { id: string }; -// credential in-line, no wrapper, arbitrary data -export type GetCredentialReply = {}; - -export const GET_DATACLIP = 'fetch:dataclip'; -export type GetDataclipPayload = { id: string }; -export type GetDataClipReply = Uint8Array; // represents a json string Attempt - -export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp -export type AttemptStartPayload = void; // no payload -export type AttemptStartReply = void; // no payload - -export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats -export type AttemptCompletePayload = { - final_dataclip_id: string; - status: 'success' | 'fail' | 'crash' | 'timeout'; - stats?: any; -}; // TODO dataclip -> result? output_dataclip? -export type AttemptCompleteReply = undefined; - -export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time -export type AttemptLogPayload = { - message: Array; - timestamp: number; - attempt_id: string; - level?: string; - source?: string; // namespace - job_id?: string; - run_id?: string; -}; -export type AttemptLogReply = void; - -export const RUN_START = 'run:start'; -export type RunStart = { - job_id: string; - run_id: string; - attempt_id?: string; - input_dataclip_id?: string; //hmm -}; -export type RunStartReply = void; - -export const RUN_COMPLETE = 'run:complete'; -export type RunCompletePayload = { - attempt_id?: string; - job_id: string; - run_id: string; - output_dataclip?: string; - output_dataclip_id?: string; -}; -export type RunCompleteReply = void; diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index c2fa36c4b..4895e77ca 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -7,11 +7,11 @@ import createLogger, { LogLevel, Logger, } from '@openfn/logger'; +import type { AttemptLogPayload } from '@openfn/ws-worker'; import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; import { Attempt } from './types'; -import { AttemptLogPayload } from './events'; export type AttemptState = { status: 'queued' | 'started' | 'complete'; diff --git a/packages/lightning-mock/test/events/attempt-start.test.ts b/packages/lightning-mock/test/events/attempt-start.test.ts new file mode 100644 index 000000000..8046b2b8c --- /dev/null +++ b/packages/lightning-mock/test/events/attempt-start.test.ts @@ -0,0 +1,62 @@ +import test from 'ava'; + +import { setup } from '../util'; +import { attempts } from '../data'; +import { ATTEMPT_START } from '../../src/events'; + +let server; +let client; + +const port = 5555; + +const attempt1 = attempts['attempt-1']; + +type Channel = any; // TODO + +test.before(async () => ({ server, client } = await setup(port))); + +const join = (attemptId: string): Promise => + new Promise((done, reject) => { + const channel = client.channel(`attempt:${attemptId}`, { token: 'a.b.c' }); + channel + .join() + .receive('ok', () => { + done(channel); + }) + .receive('error', (err) => { + reject(new Error(err)); + }); + }); + +test.serial('acknowledge attempt:start', async (t) => { + return new Promise(async (done) => { + server.registerAttempt(attempt1); + server.startAttempt(attempt1.id); + + const event = {}; + + const channel = await join(attempt1.id); + + channel.push(ATTEMPT_START, event).receive('ok', () => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +// TODO idk how much sense this makes as we have to join the channel first? +// I guess it covers a case where get in the channel but then something goes wrong +// like maybe we send two starts, one after completion +test.serial('reject attempt:start for an unknown attempt', async (t) => { + return new Promise(async (done) => { + const event = {}; + + // Note that the mock is currently lenient here + const channel = await join(attempt1.id); + + channel.push(ATTEMPT_START, event).receive('ok', () => { + t.pass('event acknowledged'); + done(); + }); + }); +}); From 4f4f86bb76316ba6af37598b0226a63d31af34df Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 14:34:09 +0000 Subject: [PATCH 07/45] worker: Refactor types to proper case --- packages/ws-worker/src/api/claim.ts | 6 +-- packages/ws-worker/src/api/execute.ts | 20 +++++----- packages/ws-worker/src/channels/attempt.ts | 4 +- packages/ws-worker/src/events.ts | 46 ++++++++++------------ packages/ws-worker/src/index.ts | 2 + packages/ws-worker/src/server.ts | 6 +-- 6 files changed, 41 insertions(+), 43 deletions(-) diff --git a/packages/ws-worker/src/api/claim.ts b/packages/ws-worker/src/api/claim.ts index 905bf6fd3..fa0cd9b0b 100644 --- a/packages/ws-worker/src/api/claim.ts +++ b/packages/ws-worker/src/api/claim.ts @@ -1,5 +1,5 @@ import { Logger, createMockLogger } from '@openfn/logger'; -import { CLAIM, CLAIM_PAYLOAD, CLAIM_REPLY } from '../events'; +import { CLAIM, ClaimPayload, ClaimReply } from '../events'; import type { ServerApp } from '../server'; @@ -15,8 +15,8 @@ const claim = (app: ServerApp, logger: Logger = mockLogger, maxWorkers = 5) => { logger.debug('requesting attempt...'); app.channel - .push(CLAIM, { demand: 1 }) - .receive('ok', ({ attempts }: CLAIM_REPLY) => { + .push(CLAIM, { demand: 1 }) + .receive('ok', ({ attempts }: ClaimReply) => { logger.debug(`pulled ${attempts.length} attempts`); // TODO what if we get here after we've been cancelled? // the events have already been claimed... diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 5cf804bb8..f072d248c 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -2,17 +2,17 @@ import crypto from 'node:crypto'; import { ATTEMPT_COMPLETE, - ATTEMPT_COMPLETE_PAYLOAD, + AttemptCompletePayload, ATTEMPT_LOG, - ATTEMPT_LOG_PAYLOAD, + AttemptLogPayload, ATTEMPT_START, - ATTEMPT_START_PAYLOAD, + AttemptStartPayload, GET_CREDENTIAL, GET_DATACLIP, RUN_COMPLETE, RunCompletePayload, RUN_START, - RUN_START_PAYLOAD, + RunStartPayload, } from '../events'; import { AttemptOptions, Channel, AttemptState } from '../types'; import { getWithReply, stringify, createAttemptState } from '../util'; @@ -168,7 +168,7 @@ export function onJobStart({ channel, state }: Context, event: any) { const input_dataclip_id = state.inputDataclips[event.jobId]; - return sendEvent(channel, RUN_START, { + return sendEvent(channel, RUN_START, { run_id: state.activeRun!, job_id: state.activeJob!, input_dataclip_id, @@ -255,7 +255,7 @@ export function onWorkflowStart( { channel }: Context, _event: WorkflowStartPayload ) { - return sendEvent(channel, ATTEMPT_START); + return sendEvent(channel, ATTEMPT_START); } export async function onWorkflowComplete( @@ -266,7 +266,7 @@ export async function onWorkflowComplete( // Especially not in parallelisation const result = state.dataclips[state.lastDataclipId!]; const reason = calculateAttemptExitReason(state); - await sendEvent(channel, ATTEMPT_COMPLETE, { + await sendEvent(channel, ATTEMPT_COMPLETE, { final_dataclip_id: state.lastDataclipId!, ...reason, }); @@ -286,7 +286,7 @@ export async function onWorkflowError( try { // Ok, let's try that, let's just generate a reason from the event const reason = calculateJobExitReason('', { data: {} }, event); - await sendEvent(channel, ATTEMPT_COMPLETE, { + await sendEvent(channel, ATTEMPT_COMPLETE, { final_dataclip_id: state.lastDataclipId!, ...reason, }); @@ -302,7 +302,7 @@ export function onJobLog({ channel, state }: Context, event: JSONLog) { const timeInMicroseconds = BigInt(event.time) / BigInt(1e3); // lightning-friendly log object - const log: ATTEMPT_LOG_PAYLOAD = { + const log: AttemptLogPayload = { attempt_id: state.plan.id!, message: event.message, source: event.name, @@ -314,7 +314,7 @@ export function onJobLog({ channel, state }: Context, event: JSONLog) { log.run_id = state.activeRun; } - return sendEvent(channel, ATTEMPT_LOG, log); + return sendEvent(channel, ATTEMPT_LOG, log); } export async function loadDataclip(channel: Channel, stateId: string) { diff --git a/packages/ws-worker/src/channels/attempt.ts b/packages/ws-worker/src/channels/attempt.ts index a6b2ea095..42a2fd765 100644 --- a/packages/ws-worker/src/channels/attempt.ts +++ b/packages/ws-worker/src/channels/attempt.ts @@ -2,7 +2,7 @@ import convertAttempt from '../util/convert-attempt'; import { getWithReply } from '../util'; import { Attempt, AttemptOptions, Channel, Socket } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; -import { GET_ATTEMPT } from '../events'; +import { GET_ATTEMPT, GetAttemptReply } from '../events'; import type { Logger } from '@openfn/logger'; @@ -52,7 +52,7 @@ export default joinAttemptChannel; export async function loadAttempt(channel: Channel) { // first we get the attempt body through the socket - const attemptBody = await getWithReply(channel, GET_ATTEMPT); + const attemptBody = await getWithReply(channel, GET_ATTEMPT); // then we generate the execution plan return convertAttempt(attemptBody as Attempt); } diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index b01fa61e2..b4c9955d1 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -2,57 +2,53 @@ import { Attempt, ExitReason } from './types'; export const CLAIM = 'claim'; -// this is the lightning payload -// TODO why are the types in all caps...? -export type CLAIM_PAYLOAD = { demand?: number }; -export type CLAIM_REPLY = { attempts: Array }; -export type CLAIM_ATTEMPT = { id: string; token: string }; +export type ClaimPayload = { demand?: number }; +export type ClaimReply = { attempts: Array }; +export type ClaimAttempt = { id: string; token: string }; export const GET_ATTEMPT = 'fetch:attempt'; -export type GET_ATTEMPT_PAYLOAD = void; // no payload -export type GET_ATTEMPT_REPLY = Attempt; +export type GetAttemptPayload = void; // no payload +export type GetAttemptReply = Attempt; export const GET_CREDENTIAL = 'fetch:credential'; -export type GET_CREDENTIAL_PAYLOAD = { id: string }; +export type GetCredentialPayload = { id: string }; // credential in-line, no wrapper, arbitrary data -export type GET_CREDENTIAL_REPLY = {}; +export type GetCredentialReply = {}; export const GET_DATACLIP = 'fetch:dataclip'; -export type GET_DATACLIP_PAYLOAD = { id: string }; -export type GET_DATACLIP_REPLY = Uint8Array; // represents a json string Attempt +export type GetDataclipPayload = { id: string }; +export type GetDataClipReply = Uint8Array; // represents a json string Attempt export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp -export type ATTEMPT_START_PAYLOAD = void; // no payload -export type ATTEMPT_START_REPLY = void; // no payload +export type AttemptStartPayload = void; // no payload +export type AttemptStartReply = void; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats -export type ATTEMPT_COMPLETE_PAYLOAD = { - final_dataclip_id: string; - status: 'success' | 'fail' | 'crash' | 'timeout'; - stats?: any; -}; // TODO dataclip -> result? output_dataclip? -export type ATTEMPT_COMPLETE_REPLY = undefined; +export type AttemptCompletePayload = ExitReason & { + final_dataclip_id?: string; // TODO this will be removed soon +}; +export type AttemptCompleteReply = undefined; export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time -export type ATTEMPT_LOG_PAYLOAD = { +export type AttemptLogPayload = { message: Array; - timestamp: string; // Tiemstamp in microseconds (13 digits) + timestamp: string; attempt_id: string; level?: string; source?: string; // namespace job_id?: string; run_id?: string; }; -export type ATTEMPT_LOG_REPLY = void; +export type AttemptLogReply = void; export const RUN_START = 'run:start'; -export type RUN_START_PAYLOAD = { +export type RunStartPayload = { job_id: string; run_id: string; attempt_id?: string; input_dataclip_id?: string; }; -export type RUN_START_REPLY = void; +export type RunStartReply = void; export const RUN_COMPLETE = 'run:complete'; export type RunCompletePayload = ExitReason & { @@ -61,6 +57,6 @@ export type RunCompletePayload = ExitReason & { run_id: string; output_dataclip?: string; output_dataclip_id?: string; - next_job_ids: string[]; + //next_job_ids: string[]; // ? }; export type RunCompleteReply = void; diff --git a/packages/ws-worker/src/index.ts b/packages/ws-worker/src/index.ts index e2d703ce5..1e2b09981 100644 --- a/packages/ws-worker/src/index.ts +++ b/packages/ws-worker/src/index.ts @@ -1,3 +1,5 @@ import createServer from './server'; export default createServer; + +export * from './events'; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 284237ecb..a2df186bc 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -13,7 +13,7 @@ import connectToWorkerQueue from './channels/worker-queue'; import type { RuntimeEngine } from '@openfn/engine-multi'; import type { Socket, Channel } from './types'; -import type { CLAIM_ATTEMPT } from './events'; +import type { ClaimAttempt } from './events'; type ServerOptions = { maxWorkflows?: number; @@ -37,7 +37,7 @@ export interface ServerApp extends Koa { channel: Channel; workflows: Record; - execute: ({ id, token }: CLAIM_ATTEMPT) => Promise; + execute: ({ id, token }: ClaimAttempt) => Promise; destroy: () => void; killWorkloop?: () => void; } @@ -137,7 +137,7 @@ function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { logger.success(`ws-worker ${app.id} listening on ${port}`); // TODO this probably needs to move into ./api/ somewhere - app.execute = async ({ id, token }: CLAIM_ATTEMPT) => { + app.execute = async ({ id, token }: ClaimAttempt) => { if (app.socket) { app.workflows[id] = true; From 24cdaabf16f27f24119d4cfa0dcc5e2e2537b2e8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 15:07:40 +0000 Subject: [PATCH 08/45] lightning-mock: tests and hardening for attempt:start --- packages/lightning-mock/src/api-dev.ts | 41 ++++----------- packages/lightning-mock/src/api-sockets.ts | 23 +++++++-- packages/lightning-mock/src/server.ts | 9 ++-- packages/lightning-mock/src/types.ts | 32 ++++++++++++ .../test/events/attempt-start.test.ts | 50 +++++++++++++++---- packages/lightning-mock/test/util.ts | 4 +- 6 files changed, 106 insertions(+), 53 deletions(-) diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index 9a7662eb2..a8f0e1fa0 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -2,48 +2,20 @@ * This module sets up a bunch of dev-only APIs * These are not intended to be reflected in Lightning itself */ -import Koa from 'koa'; import Router from '@koa/router'; import { Logger } from '@openfn/logger'; import crypto from 'node:crypto'; import { ATTEMPT_COMPLETE, AttemptCompletePayload } from '@openfn/ws-worker'; -import { Attempt } from './types'; +import { Attempt, DevServer, LightningEvents } from './types'; import { ServerState } from './server'; -type LightningEvents = 'log' | 'attempt-complete'; - -type DataClip = any; - -export type DevApp = Koa & { - addCredential(id: string, cred: Credential): void; - addDataclip(id: string, data: DataClip): void; - enqueueAttempt(attempt: Attempt): void; - getAttempt(id: string): Attempt; - getCredential(id: string): Credential; - getDataclip(id: string): DataClip; - getQueueLength(): number; - getResult(attemptId: string): any; - getState(): ServerState; - on(event: LightningEvents, fn: (evt: any) => void): void; - once(event: LightningEvents, fn: (evt: any) => void): void; - onSocketEvent( - event: LightningEvents, - attemptId: string, - fn: (evt: any) => void - ): void; - registerAttempt(attempt: Attempt): void; - reset(): void; - startAttempt(id: string): any; - waitForResult(attemptId: string): Promise; -}; - type Api = { startAttempt(attemptId: string): void; }; const setupDevAPI = ( - app: DevApp, + app: DevServer, state: ServerState, logger: Logger, api: Api @@ -147,7 +119,7 @@ const setupDevAPI = ( // Set up some rest endpoints // Note that these are NOT prefixed -const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { +const setupRestAPI = (app: DevServer, state: ServerState, logger: Logger) => { const router = new Router(); router.post('/attempt', (ctx) => { @@ -183,7 +155,12 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { return router.routes(); }; -export default (app: DevApp, state: ServerState, logger: Logger, api: Api) => { +export default ( + app: DevServer, + state: ServerState, + logger: Logger, + api: Api +) => { setupDevAPI(app, state, logger, api); return setupRestAPI(app, state, logger); }; diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index aa5d16d1c..b5cc75b4f 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -8,6 +8,7 @@ import { extractAttemptId } from './util'; import createPheonixMockSocketServer, { DevSocket, PhoenixEvent, + PhoenixEventStatus, } from './socket-server'; import { @@ -41,6 +42,8 @@ import { import type { Server } from 'http'; import { stringify } from './util'; +import { AttemptStartPayload } from '@openfn/ws-worker'; +import { AttemptStartReply } from '@openfn/ws-worker'; // dumb cloning id // just an idea for unit tests @@ -204,16 +207,26 @@ const createSocketAPI = ( function handleStartAttempt( _state: ServerState, ws: DevSocket, - evt: PhoenixEvent + evt: PhoenixEvent ) { const { ref, join_ref, topic } = evt; - ws.reply({ + const [_, attemptId] = topic.split(':'); + let payload = { + status: 'ok' as PhoenixEventStatus, + }; + if ( + !state.pending[attemptId] || + state.pending[attemptId].status !== 'started' + ) { + payload = { + status: 'error', + }; + } + ws.reply({ ref, join_ref, topic, - payload: { - status: 'ok', - }, + payload, }); } diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index 4895e77ca..f18a4c5d3 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -11,7 +11,7 @@ import type { AttemptLogPayload } from '@openfn/ws-worker'; import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; -import { Attempt } from './types'; +import { Attempt, DevServer } from './types'; export type AttemptState = { status: 'queued' | 'started' | 'complete'; @@ -64,9 +64,11 @@ const createLightningServer = (options: LightningOptions = {}) => { events: new EventEmitter(), } as ServerState; - const app = new Koa(); + const app = new Koa() as DevServer; app.use(bodyParser()); + app.state = state; + const port = options.port || 8888; const server = app.listen(port); logger.info('Listening on ', port); @@ -89,11 +91,10 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(createDevAPI(app as any, state, logger, api)); - (app as any).destroy = () => { + app.destroy = () => { server.close(); api.close(); }; - return app; }; diff --git a/packages/lightning-mock/src/types.ts b/packages/lightning-mock/src/types.ts index f77948a64..b26bdbe4b 100644 --- a/packages/lightning-mock/src/types.ts +++ b/packages/lightning-mock/src/types.ts @@ -1,3 +1,6 @@ +import Koa from 'koa'; +import type { ServerState } from './server'; + export type Node = { id: string; body?: string; @@ -30,3 +33,32 @@ export type Attempt = { options?: Record; // TODO type the expected options }; + +export type LightningEvents = 'log' | 'attempt-complete'; + +export type DataClip = any; + +export type DevServer = Koa & { + state: ServerState; + addCredential(id: string, cred: Credential): void; + addDataclip(id: string, data: DataClip): void; + enqueueAttempt(attempt: Attempt): void; + destroy: () => void; + getAttempt(id: string): Attempt; + getCredential(id: string): Credential; + getDataclip(id: string): DataClip; + getQueueLength(): number; + getResult(attemptId: string): any; + getState(): ServerState; + on(event: LightningEvents, fn: (evt: any) => void): void; + once(event: LightningEvents, fn: (evt: any) => void): void; + onSocketEvent( + event: LightningEvents, + attemptId: string, + fn: (evt: any) => void + ): void; + registerAttempt(attempt: Attempt): void; + reset(): void; + startAttempt(id: string): any; + waitForResult(attemptId: string): Promise; +}; diff --git a/packages/lightning-mock/test/events/attempt-start.test.ts b/packages/lightning-mock/test/events/attempt-start.test.ts index 8046b2b8c..f9ab8a073 100644 --- a/packages/lightning-mock/test/events/attempt-start.test.ts +++ b/packages/lightning-mock/test/events/attempt-start.test.ts @@ -1,8 +1,9 @@ import test from 'ava'; +import crypto from 'node:crypto'; import { setup } from '../util'; import { attempts } from '../data'; -import { ATTEMPT_START } from '../../src/events'; +import { ATTEMPT_START } from '@openfn/ws-worker'; let server; let client; @@ -15,6 +16,11 @@ type Channel = any; // TODO test.before(async () => ({ server, client } = await setup(port))); +const createAttempt = () => ({ + ...attempt1, + id: crypto.randomUUID(), +}); + const join = (attemptId: string): Promise => new Promise((done, reject) => { const channel = client.channel(`attempt:${attemptId}`, { token: 'a.b.c' }); @@ -30,12 +36,13 @@ const join = (attemptId: string): Promise => test.serial('acknowledge attempt:start', async (t) => { return new Promise(async (done) => { - server.registerAttempt(attempt1); - server.startAttempt(attempt1.id); + const attempt = createAttempt(); + + server.startAttempt(attempt.id); const event = {}; - const channel = await join(attempt1.id); + const channel = await join(attempt.id); channel.push(ATTEMPT_START, event).receive('ok', () => { t.pass('event acknowledged'); @@ -44,18 +51,41 @@ test.serial('acknowledge attempt:start', async (t) => { }); }); -// TODO idk how much sense this makes as we have to join the channel first? -// I guess it covers a case where get in the channel but then something goes wrong -// like maybe we send two starts, one after completion test.serial('reject attempt:start for an unknown attempt', async (t) => { return new Promise(async (done) => { + const attempt = createAttempt(); const event = {}; + server.startAttempt(attempt.id); + // Note that the mock is currently lenient here - const channel = await join(attempt1.id); + const channel = await join(attempt.id); - channel.push(ATTEMPT_START, event).receive('ok', () => { - t.pass('event acknowledged'); + // Sneak into the server and kill the state for this attempt + delete server.state.pending[attempt.id]; + + channel.push(ATTEMPT_START, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('reject attempt:start for a completed attempt', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + const event = {}; + + server.startAttempt(attempt.id); + + // Note that the mock is currently lenient here + const channel = await join(attempt.id); + + // Sneak into the server and update the state for this attempt + server.state.pending[attempt.id].status = 'completed'; + + channel.push(ATTEMPT_START, event).receive('error', () => { + t.pass('event rejected'); done(); }); }); diff --git a/packages/lightning-mock/test/util.ts b/packages/lightning-mock/test/util.ts index a96d2c149..f19cda21d 100644 --- a/packages/lightning-mock/test/util.ts +++ b/packages/lightning-mock/test/util.ts @@ -2,11 +2,11 @@ import createLightningServer from '../src/server'; import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; +import type { DevServer } from '../src/types'; export const setup = (port: number) => { - return new Promise<{ server: any; client: any }>((done) => { + return new Promise<{ server: DevServer; client: any }>((done) => { const server = createLightningServer({ port }); - // Note that we need a token to connect, but the mock here // doesn't (yet) do any validation on that token const client = new Socket(`ws://localhost:${port}/worker`, { From e7d80279fe630eed8cb84a93471ada619b264e7d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 15:31:00 +0000 Subject: [PATCH 09/45] lightning-mock: tests and hardening for attempt:complete --- packages/lightning-mock/src/api-sockets.ts | 39 ++++- .../test/events/attempt-complete.test.ts | 161 ++++++++++++++++++ .../test/events/attempt-start.test.ts | 34 +--- packages/lightning-mock/test/util.ts | 26 ++- 4 files changed, 224 insertions(+), 36 deletions(-) create mode 100644 packages/lightning-mock/test/events/attempt-complete.test.ts diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index b5cc75b4f..1db1dcaec 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -300,13 +300,16 @@ const createSocketAPI = ( evt: PhoenixEvent, attemptId: string ) { - const { ref, join_ref, topic, payload } = evt; - const { final_dataclip_id } = payload; + const { ref, join_ref, topic } = evt; + const { final_dataclip_id, reason, error_type, error_message, ...rest } = + evt.payload; logger?.info('Completed attempt ', attemptId); logger?.debug(final_dataclip_id); state.pending[attemptId].status = 'complete'; + + // TODO we'll remove this stuff soon if (!state.results[attemptId]) { state.results[attemptId] = { state: null, workerId: 'mock' }; } @@ -314,13 +317,39 @@ const createSocketAPI = ( state.results[attemptId].state = state.dataclips[final_dataclip_id]; } + let payload: any = { + status: 'ok', + }; + + const invalidKeys = Object.keys(rest); + + if (!reason) { + payload = { + status: 'error', + response: `No exit reason`, + }; + } else if (!/^(success|fail|crash|exception|kill)$/.test(reason)) { + payload = { + status: 'error', + response: `Unrecognised reason ${reason}`, + }; + } else if (invalidKeys.length) { + payload = { + status: 'error', + response: `Unexpected keys: ${invalidKeys.join(',')}`, + }; + } else if (reason === 'success' && (error_type || error_message)) { + payload = { + status: 'error', + response: `Inconsistent reason (success and error type or message)`, + }; + } + ws.reply({ ref, join_ref, topic, - payload: { - status: 'ok', - }, + payload, }); } diff --git a/packages/lightning-mock/test/events/attempt-complete.test.ts b/packages/lightning-mock/test/events/attempt-complete.test.ts new file mode 100644 index 000000000..2f0f9b791 --- /dev/null +++ b/packages/lightning-mock/test/events/attempt-complete.test.ts @@ -0,0 +1,161 @@ +import test from 'ava'; +import { ATTEMPT_COMPLETE } from '@openfn/ws-worker'; + +import { join, setup, createAttempt } from '../util'; + +let server; +let client; + +const port = 5501; + +test.before(async () => ({ server, client } = await setup(port))); + +test.serial('acknowledge valid message', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + error_type: null, + error_message: null, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('ok', () => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +test.serial('set server state to complete', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + error_type: null, + error_message: null, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('ok', () => { + t.is(server.state.pending[attempt.id].status, 'complete'); + done(); + }); + }); +}); + +test.serial('error if no reason', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: null, + error_type: null, + error_message: null, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error reason:success and an error', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + error_type: 'OOM', + error_message: 'out of memory', + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if surplus keys', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + error_type: null, + error_message: null, + err: true, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if unknown reason', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'swish', + error_type: null, + error_message: null, + err: true, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if unknown reason 2', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'crassh', + error_type: null, + error_message: null, + err: true, + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); diff --git a/packages/lightning-mock/test/events/attempt-start.test.ts b/packages/lightning-mock/test/events/attempt-start.test.ts index f9ab8a073..19425842e 100644 --- a/packages/lightning-mock/test/events/attempt-start.test.ts +++ b/packages/lightning-mock/test/events/attempt-start.test.ts @@ -1,39 +1,15 @@ import test from 'ava'; -import crypto from 'node:crypto'; -import { setup } from '../util'; -import { attempts } from '../data'; +import { join, setup, createAttempt } from '../util'; import { ATTEMPT_START } from '@openfn/ws-worker'; let server; let client; -const port = 5555; - -const attempt1 = attempts['attempt-1']; - -type Channel = any; // TODO +const port = 5500; test.before(async () => ({ server, client } = await setup(port))); -const createAttempt = () => ({ - ...attempt1, - id: crypto.randomUUID(), -}); - -const join = (attemptId: string): Promise => - new Promise((done, reject) => { - const channel = client.channel(`attempt:${attemptId}`, { token: 'a.b.c' }); - channel - .join() - .receive('ok', () => { - done(channel); - }) - .receive('error', (err) => { - reject(new Error(err)); - }); - }); - test.serial('acknowledge attempt:start', async (t) => { return new Promise(async (done) => { const attempt = createAttempt(); @@ -42,7 +18,7 @@ test.serial('acknowledge attempt:start', async (t) => { const event = {}; - const channel = await join(attempt.id); + const channel = await join(client, attempt.id); channel.push(ATTEMPT_START, event).receive('ok', () => { t.pass('event acknowledged'); @@ -59,7 +35,7 @@ test.serial('reject attempt:start for an unknown attempt', async (t) => { server.startAttempt(attempt.id); // Note that the mock is currently lenient here - const channel = await join(attempt.id); + const channel = await join(client, attempt.id); // Sneak into the server and kill the state for this attempt delete server.state.pending[attempt.id]; @@ -79,7 +55,7 @@ test.serial('reject attempt:start for a completed attempt', async (t) => { server.startAttempt(attempt.id); // Note that the mock is currently lenient here - const channel = await join(attempt.id); + const channel = await join(client, attempt.id); // Sneak into the server and update the state for this attempt server.state.pending[attempt.id].status = 'completed'; diff --git a/packages/lightning-mock/test/util.ts b/packages/lightning-mock/test/util.ts index f19cda21d..6472a23c5 100644 --- a/packages/lightning-mock/test/util.ts +++ b/packages/lightning-mock/test/util.ts @@ -1,8 +1,12 @@ -import createLightningServer from '../src/server'; - import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; +import crypto from 'node:crypto'; + +import createLightningServer from '../src/server'; import type { DevServer } from '../src/types'; +import { attempts } from './data'; + +type Channel = any; // TODO export const setup = (port: number) => { return new Promise<{ server: DevServer; client: any }>((done) => { @@ -20,3 +24,21 @@ export const setup = (port: number) => { client.connect(); }); }; + +export const join = (client: any, attemptId: string): Promise => + new Promise((done, reject) => { + const channel = client.channel(`attempt:${attemptId}`, { token: 'a.b.c' }); + channel + .join() + .receive('ok', () => { + done(channel); + }) + .receive('error', (err) => { + reject(new Error(err)); + }); + }); + +export const createAttempt = () => ({ + ...attempts['attempt-1'], + id: crypto.randomUUID(), +}); From 496808701754cd6939f0249e107363590446a2a3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 15:43:48 +0000 Subject: [PATCH 10/45] lightning-mock: tests and hardening for run:start --- packages/lightning-mock/src/api-sockets.ts | 31 ++++- packages/lightning-mock/src/server.ts | 4 + packages/lightning-mock/test/data.ts | 9 +- .../test/events/run-start.test.ts | 118 ++++++++++++++++++ 4 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 packages/lightning-mock/test/events/run-start.test.ts diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 1db1dcaec..17d61ba77 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -101,6 +101,7 @@ const createSocketAPI = ( state.pending[attemptId] = { status: 'started', logs: [], + runs: {}, }; const wrap = ( @@ -359,16 +360,40 @@ const createSocketAPI = ( evt: PhoenixEvent ) { const { ref, join_ref, topic } = evt; + const { run_id, job_id, input_dataclip_id } = evt.payload; + + const [_, attemptId] = topic.split(':'); if (!state.dataclips) { state.dataclips = {}; } + state.pending[attemptId].runs[job_id] = run_id; + + let payload: any = { + status: 'ok', + }; + + if (!run_id) { + payload = { + status: 'error', + response: 'no run_id', + }; + } else if (!job_id) { + payload = { + status: 'error', + response: 'no job_id', + }; + } else if (!input_dataclip_id) { + payload = { + status: 'error', + response: 'no input_dataclip_id', + }; + } + ws.reply({ ref, join_ref, topic, - payload: { - status: 'ok', - }, + payload, }); } diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index f18a4c5d3..989944a1d 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -13,9 +13,13 @@ import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; import { Attempt, DevServer } from './types'; +type RunId = string; +type JobId = string; + export type AttemptState = { status: 'queued' | 'started' | 'complete'; logs: AttemptLogPayload[]; + runs: Record; }; export type ServerState = { diff --git a/packages/lightning-mock/test/data.ts b/packages/lightning-mock/test/data.ts index e6e773be6..6773f51c6 100644 --- a/packages/lightning-mock/test/data.ts +++ b/packages/lightning-mock/test/data.ts @@ -14,18 +14,15 @@ export const dataclips = { export const attempts = { 'attempt-1': { id: 'attempt-1', - // TODO how should this be structure? - input: { - data: 'd', - }, + dataclip_id: 'x', triggers: [], edges: [], jobs: [ { - id: 'job-1', + id: 'a', adaptor: '@openfn/language-common@1.0.0', body: 'fn(a => a)', - credential: 'a', + credential: 'abc', }, ], }, diff --git a/packages/lightning-mock/test/events/run-start.test.ts b/packages/lightning-mock/test/events/run-start.test.ts new file mode 100644 index 000000000..9ac20edb4 --- /dev/null +++ b/packages/lightning-mock/test/events/run-start.test.ts @@ -0,0 +1,118 @@ +import test from 'ava'; +import { RUN_START } from '@openfn/ws-worker'; + +import { join, setup, createAttempt } from '../util'; + +let server; +let client; + +const port = 5501; + +test.before(async () => ({ server, client } = await setup(port))); + +test.serial('acknowledge valid message', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + job_id: 'a', + run_id: 'r:a', + input_dataclip_id: 'x', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_START, event).receive('ok', () => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +test.serial('save run id to state', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + job_id: 'a', + run_id: 'r:a', + input_dataclip_id: 'x', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_START, event).receive('ok', () => { + t.deepEqual(server.state.pending[attempt.id].runs, { + [event.job_id]: event.run_id, + }); + done(); + }); + }); +}); + +test.serial('error if no run_id', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + job_id: 'a', + run_id: undefined, + input_dataclip_id: 'x', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_START, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no job_id', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + job_id: undefined, + run_id: 'r:a', + input_dataclip_id: 'x', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_START, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no input_dataclip_id', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + job_id: 'a', + run_id: 'r:a', + input_dataclip_id: undefined, + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_START, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); From e1f850fced0c5c170633204810dd59415ef8d98a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:03:44 +0000 Subject: [PATCH 11/45] lightning-mock: tests and hardening for run:complete --- packages/lightning-mock/src/api-sockets.ts | 61 +++++---- .../test/events/run-complete.test.ts | 116 ++++++++++++++++++ 2 files changed, 153 insertions(+), 24 deletions(-) create mode 100644 packages/lightning-mock/test/events/run-complete.test.ts diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 17d61ba77..6bf790b80 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -54,6 +54,27 @@ const clone = (state: ServerState) => { const enc = new TextEncoder(); +const validateReasons = (evt: any) => { + const { reason, error_message, error_type } = evt; + if (!reason) { + return { + status: 'error', + response: `No exit reason`, + }; + } else if (!/^(success|fail|crash|exception|kill)$/.test(reason)) { + return { + status: 'error', + response: `Unrecognised reason ${reason}`, + }; + } else if (reason === 'success' && (error_type || error_message)) { + return { + status: 'error', + response: `Inconsistent reason (success and error type or message)`, + }; + } + return { status: 'ok' }; +}; + // this new API is websocket based // Events map to handlers // can I even implement this in JS? Not with pheonix anyway. hmm. @@ -318,32 +339,14 @@ const createSocketAPI = ( state.results[attemptId].state = state.dataclips[final_dataclip_id]; } - let payload: any = { - status: 'ok', - }; + let payload: any = validateReasons(evt.payload); const invalidKeys = Object.keys(rest); - - if (!reason) { - payload = { - status: 'error', - response: `No exit reason`, - }; - } else if (!/^(success|fail|crash|exception|kill)$/.test(reason)) { - payload = { - status: 'error', - response: `Unrecognised reason ${reason}`, - }; - } else if (invalidKeys.length) { + if (payload.status !== 'ok' && invalidKeys.length) { payload = { status: 'error', response: `Unexpected keys: ${invalidKeys.join(',')}`, }; - } else if (reason === 'success' && (error_type || error_message)) { - payload = { - status: 'error', - response: `Inconsistent reason (success and error type or message)`, - }; } ws.reply({ @@ -405,11 +408,23 @@ const createSocketAPI = ( const { ref, join_ref, topic } = evt; const { output_dataclip_id, output_dataclip } = evt.payload; - if (output_dataclip_id) { + let payload: any = validateReasons(evt.payload); + + if (!output_dataclip) { + payload = { + status: 'error', + response: 'no output_dataclip', + }; + } else if (output_dataclip_id) { if (!state.dataclips) { state.dataclips = {}; } state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip!); + } else { + payload = { + status: 'error', + response: 'no output_dataclip_id', + }; } // be polite and acknowledge the event @@ -417,9 +432,7 @@ const createSocketAPI = ( ref, join_ref, topic, - payload: { - status: 'ok', - }, + payload, }); } }; diff --git a/packages/lightning-mock/test/events/run-complete.test.ts b/packages/lightning-mock/test/events/run-complete.test.ts new file mode 100644 index 000000000..7c9bcdbef --- /dev/null +++ b/packages/lightning-mock/test/events/run-complete.test.ts @@ -0,0 +1,116 @@ +import test from 'ava'; +import { RUN_COMPLETE } from '@openfn/ws-worker'; + +import { join, setup, createAttempt } from '../util'; + +let server; +let client; + +const port = 5501; + +test.before(async () => ({ server, client } = await setup(port))); + +test.serial('acknowledge valid message', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + output_dataclip: JSON.stringify({ x: 22 }), + output_dataclip_id: 't', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_COMPLETE, event).receive('ok', (evt) => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +test.serial('save dataclip id to state', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + output_dataclip: JSON.stringify({ x: 22 }), + output_dataclip_id: 't', + }; + + const channel = await join(client, attempt.id); + + channel.push(RUN_COMPLETE, event).receive('ok', () => { + t.deepEqual(server.state.dataclips.t, JSON.parse(event.output_dataclip)); + done(); + }); + }); +}); + +test.serial('error if no dataclip', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + output_dataclip: null, + output_dataclip_id: 't', + }; + const channel = await join(client, attempt.id); + + channel.push(RUN_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no dataclip_d', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: 'success', + output_dataclip: JSON.stringify({ x: 22 }), + output_dataclip_id: undefined, + }; + const channel = await join(client, attempt.id); + + channel.push(RUN_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no reason', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + reason: undefined, + output_dataclip: JSON.stringify({ x: 22 }), + output_dataclip_id: undefined, + }; + const channel = await join(client, attempt.id); + + channel.push(RUN_COMPLETE, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +// reason validation code is shared with attempt:complete +// It's fine not to test further here From 306567d3e1dfb216edcfb11ee466c4779da8c1c1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:12:15 +0000 Subject: [PATCH 12/45] lightning-mock: tests and hardening for log --- packages/lightning-mock/src/api-sockets.ts | 26 ++- .../test/channels/attempt.test.ts | 23 --- .../lightning-mock/test/events/log.test.ts | 171 ++++++++++++++++++ packages/lightning-mock/test/server.test.ts | 2 +- 4 files changed, 192 insertions(+), 30 deletions(-) create mode 100644 packages/lightning-mock/test/events/log.test.ts diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 6bf790b80..d576e1357 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -301,18 +301,32 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, join_ref, topic, payload } = evt; - const { attempt_id: attemptId } = payload; + const { ref, join_ref, topic } = evt; + const { attempt_id: attemptId } = evt.payload; - state.pending[attemptId].logs.push(payload); + state.pending[attemptId].logs.push(evt.payload); + + let payload: any = { + status: 'ok', + }; + + if ( + !evt.payload.message || + !evt.payload.source || + !evt.payload.timestamp || + !evt.payload.level + ) { + payload = { + status: 'error', + response: 'Missing property on log', + }; + } ws.reply({ ref, join_ref, topic, - payload: { - status: 'ok', - }, + payload, }); } diff --git a/packages/lightning-mock/test/channels/attempt.test.ts b/packages/lightning-mock/test/channels/attempt.test.ts index 2fce1e11e..3ef672c2d 100644 --- a/packages/lightning-mock/test/channels/attempt.test.ts +++ b/packages/lightning-mock/test/channels/attempt.test.ts @@ -99,29 +99,6 @@ test.serial('complete an attempt through the attempt channel', async (t) => { }); }); -test.serial('logs are saved and acknowledged', async (t) => { - return new Promise(async (done) => { - server.registerAttempt(attempt1); - server.startAttempt(attempt1.id); - - const log = { - attempt_id: attempt1.id, - level: 'info', - name: 'R/T', - message: ['Did the thing'], - time: new Date().getTime(), - } as JSONLog; - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_LOG, log).receive('ok', () => { - const { pending } = server.getState(); - const [savedLog] = pending[attempt1.id].logs; - t.deepEqual(savedLog, log); - done(); - }); - }); -}); - test.serial('unsubscribe after attempt complete', async (t) => { return new Promise(async (done) => { const a = attempt1; diff --git a/packages/lightning-mock/test/events/log.test.ts b/packages/lightning-mock/test/events/log.test.ts new file mode 100644 index 000000000..af4089e7b --- /dev/null +++ b/packages/lightning-mock/test/events/log.test.ts @@ -0,0 +1,171 @@ +import test from 'ava'; +import { ATTEMPT_LOG } from '@openfn/ws-worker'; + +import { join, setup, createAttempt } from '../util'; + +let server; +let client; + +const port = 5501; + +test.before(async () => ({ server, client } = await setup(port))); + +test.serial('acknowledge valid message (attempt log)', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + message: 'blah', + level: 'info', + source: 'R/T', + timestamp: '123', + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('ok', (evt) => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +test.serial('acknowledge valid message (job log)', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + job_id: 'a', + message: 'blah', + level: 'info', + source: 'R/T', + timestamp: '123', + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('ok', (evt) => { + t.pass('event acknowledged'); + done(); + }); + }); +}); + +test.serial('save log to state', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + job_id: 'a', + message: 'blah', + level: 'info', + source: 'R/T', + timestamp: '123', + }; + + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('ok', () => { + const { pending } = server.getState(); + const [savedLog] = pending[attempt.id].logs; + t.deepEqual(savedLog, event); + done(); + }); + }); +}); + +test.serial('error if no message', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + job_id: 'a', + level: 'info', + source: 'R/T', + timestamp: '123', + }; + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no source', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + job_id: 'a', + message: 'blah', + level: 'info', + timestamp: '123', + }; + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no timestamp', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + message: 'blah', + level: 'info', + source: 'R/T', + }; + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); + +test.serial('error if no level', async (t) => { + return new Promise(async (done) => { + const attempt = createAttempt(); + + server.startAttempt(attempt.id); + + const event = { + attempt_id: attempt.id, + job_id: 'a', + message: 'blah', + source: 'R/T', + timestamp: '123', + }; + const channel = await join(client, attempt.id); + + channel.push(ATTEMPT_LOG, event).receive('error', () => { + t.pass('event rejected'); + done(); + }); + }); +}); diff --git a/packages/lightning-mock/test/server.test.ts b/packages/lightning-mock/test/server.test.ts index b42c21061..fab38d313 100644 --- a/packages/lightning-mock/test/server.test.ts +++ b/packages/lightning-mock/test/server.test.ts @@ -3,7 +3,7 @@ import test from 'ava'; import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; -import { setup } from './util'; +import { createAttempt, setup } from './util'; import type { Attempt } from '../src/types'; let server; From 91b715e7f3742c7883f7e7685060b26ab8dd20a8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:12:51 +0000 Subject: [PATCH 13/45] worker: remove unused event key --- packages/ws-worker/src/events.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index b4c9955d1..390590b3e 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -21,7 +21,7 @@ export type GetDataClipReply = Uint8Array; // represents a json string Attempt export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export type AttemptStartPayload = void; // no payload -export type AttemptStartReply = void; // no payload +export type AttemptStartReply = {}; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats export type AttemptCompletePayload = ExitReason & { @@ -57,6 +57,5 @@ export type RunCompletePayload = ExitReason & { run_id: string; output_dataclip?: string; output_dataclip_id?: string; - //next_job_ids: string[]; // ? }; export type RunCompleteReply = void; From 9bf94f83a3bc3bcbbc08f2146b63f0fc8bfd91af Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:15:34 +0000 Subject: [PATCH 14/45] lightning-mock: add a changeset for the hell of it --- .changeset/tidy-trains-care.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/tidy-trains-care.md diff --git a/.changeset/tidy-trains-care.md b/.changeset/tidy-trains-care.md new file mode 100644 index 000000000..55e0f2bfe --- /dev/null +++ b/.changeset/tidy-trains-care.md @@ -0,0 +1,5 @@ +--- +'@openfn/lightning-mock': minor +--- + +Add validation to endpoints From 957b37b8e71fe16877761bff39449c12b7804c39 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:16:06 +0000 Subject: [PATCH 15/45] version bump --- .changeset/tidy-trains-care.md | 5 ----- integration-tests/worker/CHANGELOG.md | 8 ++++++++ integration-tests/worker/package.json | 2 +- packages/lightning-mock/CHANGELOG.md | 10 ++++++++++ packages/lightning-mock/package.json | 2 +- pnpm-lock.yaml | 3 +++ 6 files changed, 23 insertions(+), 7 deletions(-) delete mode 100644 .changeset/tidy-trains-care.md diff --git a/.changeset/tidy-trains-care.md b/.changeset/tidy-trains-care.md deleted file mode 100644 index 55e0f2bfe..000000000 --- a/.changeset/tidy-trains-care.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/lightning-mock': minor ---- - -Add validation to endpoints diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 1d32859db..53cab32f2 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,13 @@ # @openfn/integration-tests-worker +## 1.0.18 + +### Patch Changes + +- Updated dependencies [9bf94f8] + - @openfn/lightning-mock@1.1.0 + - @openfn/ws-worker@0.2.7 + ## 1.0.17 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 996c754fa..d8061eb43 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.17", + "version": "1.0.18", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index 252d9a97b..3b6901598 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,15 @@ # @openfn/lightning-mock +## 1.1.0 + +### Minor Changes + +- 9bf94f8: Add validation to endpoints + +### Patch Changes + +- @openfn/ws-worker@0.2.7 + ## 1.0.12 ### Patch Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index c7c549b25..86fed0b59 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.0.12", + "version": "1.1.0", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3c352c711..21705ef58 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -430,6 +430,9 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime + '@openfn/ws-worker': + specifier: workspace:^ + version: link:../ws-worker '@types/koa-logger': specifier: ^3.1.2 version: 3.1.2 From d8932a0806e33f683c9aae2fc42e1c2036cee17d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:20:04 +0000 Subject: [PATCH 16/45] Version bumps --- integration-tests/worker/CHANGELOG.md | 8 ++++++++ integration-tests/worker/package.json | 2 +- packages/lightning-mock/CHANGELOG.md | 7 +++++++ packages/lightning-mock/package.json | 2 +- packages/ws-worker/CHANGELOG.md | 6 ++++++ packages/ws-worker/package.json | 2 +- 6 files changed, 24 insertions(+), 3 deletions(-) diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 53cab32f2..3109aaf64 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,13 @@ # @openfn/integration-tests-worker +## 1.0.19 + +### Patch Changes + +- Updated dependencies + - @openfn/ws-worker@0.2.8 + - @openfn/lightning-mock@1.1.1 + ## 1.0.18 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index d8061eb43..0d0f4b60e 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.18", + "version": "1.0.19", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index 3b6901598..82e4aa585 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/lightning-mock +## 1.1.1 + +### Patch Changes + +- Updated dependencies + - @openfn/ws-worker@0.2.8 + ## 1.1.0 ### Minor Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 86fed0b59..6c001e00f 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.1.0", + "version": "1.1.1", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index 5520fe65d..544b78389 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,11 @@ # ws-worker +## 0.2.8 + +### Patch Changes + +- Tweak typings + ## 0.2.7 ### Patch Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 32c4a3aed..8c0461963 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "0.2.7", + "version": "0.2.8", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", From f2e88e71cd775c43e48862a1280ef6a2295e1b24 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 16:46:37 +0000 Subject: [PATCH 17/45] lightning-mock: fix tests and dependencies --- packages/lightning-mock/package.json | 2 +- packages/lightning-mock/src/api-dev.ts | 7 +++-- packages/lightning-mock/src/api-sockets.ts | 27 ++++++++++--------- packages/lightning-mock/src/events.ts | 10 +++++++ packages/lightning-mock/src/server.ts | 5 ++-- .../test/channels/attempt.test.ts | 15 ++++++----- .../test/events/attempt-complete.test.ts | 3 +-- .../test/events/attempt-start.test.ts | 3 +-- .../lightning-mock/test/events/log.test.ts | 2 +- .../test/events/run-complete.test.ts | 2 +- .../test/events/run-start.test.ts | 3 +-- 11 files changed, 47 insertions(+), 32 deletions(-) diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 6c001e00f..a887c48ae 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -20,7 +20,6 @@ "@openfn/engine-multi": "workspace:*", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", - "@openfn/ws-worker": "workspace:^", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", @@ -32,6 +31,7 @@ "ws": "^8.14.1" }, "devDependencies": { + "@openfn/ws-worker": "workspace:^", "@types/koa": "^2.13.5", "@types/koa-bodyparser": "^4.3.10", "@types/koa-route": "^3.2.6", diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index a8f0e1fa0..9c3af6f69 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -5,11 +5,13 @@ import Router from '@koa/router'; import { Logger } from '@openfn/logger'; import crypto from 'node:crypto'; -import { ATTEMPT_COMPLETE, AttemptCompletePayload } from '@openfn/ws-worker'; +import { ATTEMPT_COMPLETE } from './events'; -import { Attempt, DevServer, LightningEvents } from './types'; import { ServerState } from './server'; +import type { Attempt, DevServer, LightningEvents } from './types'; +import type { AttemptCompletePayload } from '@openfn/ws-worker'; + type Api = { startAttempt(attemptId: string): void; }; @@ -44,6 +46,7 @@ const setupDevAPI = ( state.pending[attempt.id] = { status: 'queued', logs: [], + runs: {}, }; state.queue.push(attempt.id); }; diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index d576e1357..334d49ef8 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -10,41 +10,42 @@ import createPheonixMockSocketServer, { PhoenixEvent, PhoenixEventStatus, } from './socket-server'; - import { ATTEMPT_COMPLETE, ATTEMPT_LOG, ATTEMPT_START, + CLAIM, + GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, + RUN_COMPLETE, + RUN_START, +} from './events'; +import type { Server } from 'http'; +import { stringify } from './util'; + +import type { + AttemptStartPayload, + AttemptStartReply, AttemptCompletePayload, AttemptCompleteReply, AttemptLogPayload, AttemptLogReply, - CLAIM, ClaimAttempt, ClaimPayload, ClaimReply, - GET_ATTEMPT, - GET_CREDENTIAL, - GET_DATACLIP, GetAttemptPayload, GetAttemptReply, GetCredentialPayload, GetCredentialReply, GetDataclipPayload, GetDataClipReply, - RUN_COMPLETE, - RUN_START, RunCompletePayload, RunCompleteReply, RunStartPayload, RunStartReply, } from '@openfn/ws-worker'; -import type { Server } from 'http'; -import { stringify } from './util'; -import { AttemptStartPayload } from '@openfn/ws-worker'; -import { AttemptStartReply } from '@openfn/ws-worker'; - // dumb cloning id // just an idea for unit tests const clone = (state: ServerState) => { @@ -356,7 +357,7 @@ const createSocketAPI = ( let payload: any = validateReasons(evt.payload); const invalidKeys = Object.keys(rest); - if (payload.status !== 'ok' && invalidKeys.length) { + if (payload.status === 'ok' && invalidKeys.length) { payload = { status: 'error', response: `Unexpected keys: ${invalidKeys.join(',')}`, diff --git a/packages/lightning-mock/src/events.ts b/packages/lightning-mock/src/events.ts index 163b5c808..4b7eebd4a 100644 --- a/packages/lightning-mock/src/events.ts +++ b/packages/lightning-mock/src/events.ts @@ -11,3 +11,13 @@ export const CONNECT = 'socket:connect'; // client left or joined a channel export const CHANNEL_JOIN = 'socket:channel-join'; export const CHANNEL_LEAVE = 'socket:channel-leave'; + +export const CLAIM = 'claim'; +export const GET_ATTEMPT = 'fetch:attempt'; +export const GET_CREDENTIAL = 'fetch:credential'; +export const GET_DATACLIP = 'fetch:dataclip'; +export const ATTEMPT_START = 'attempt:start'; +export const ATTEMPT_COMPLETE = 'attempt:complete'; +export const ATTEMPT_LOG = 'attempt:log'; +export const RUN_START = 'run:start'; +export const RUN_COMPLETE = 'run:complete'; diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index 989944a1d..f5440dedc 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -7,11 +7,12 @@ import createLogger, { LogLevel, Logger, } from '@openfn/logger'; -import type { AttemptLogPayload } from '@openfn/ws-worker'; import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; -import { Attempt, DevServer } from './types'; + +import type { AttemptLogPayload } from '@openfn/ws-worker'; +import type { Attempt, DevServer } from './types'; type RunId = string; type JobId = string; diff --git a/packages/lightning-mock/test/channels/attempt.test.ts b/packages/lightning-mock/test/channels/attempt.test.ts index 3ef672c2d..6dc0e0819 100644 --- a/packages/lightning-mock/test/channels/attempt.test.ts +++ b/packages/lightning-mock/test/channels/attempt.test.ts @@ -4,13 +4,12 @@ import { setup } from '../util'; import { attempts, credentials, dataclips } from '../data'; import { ATTEMPT_COMPLETE, - AttemptCompletePayload, - ATTEMPT_LOG, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, } from '../../src/events'; -import { JSONLog } from '@openfn/logger'; + +import { AttemptCompletePayload } from '@openfn/ws-worker'; const enc = new TextDecoder('utf-8'); @@ -89,10 +88,14 @@ test.serial('complete an attempt through the attempt channel', async (t) => { const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); channel - .push(ATTEMPT_COMPLETE, { final_dataclip_id: 'abc' }) + .push(ATTEMPT_COMPLETE, { reason: 'success', final_dataclip_id: 'abc' }) .receive('ok', () => { const { pending, results } = server.getState(); - t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); + t.deepEqual(pending[a.id], { + status: 'complete', + logs: [], + runs: {}, + }); t.deepEqual(results[a.id].state, { answer: 42 }); done(); }); @@ -106,7 +109,7 @@ test.serial('unsubscribe after attempt complete', async (t) => { server.startAttempt(a.id); const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_COMPLETE).receive('ok', () => { + channel.push(ATTEMPT_COMPLETE, { reason: 'success' }).receive('ok', () => { // After the complete event, the listener should unsubscribe to the channel // The mock will send an error to any unhandled events in that channel channel.push(ATTEMPT_COMPLETE).receive('error', () => { diff --git a/packages/lightning-mock/test/events/attempt-complete.test.ts b/packages/lightning-mock/test/events/attempt-complete.test.ts index 2f0f9b791..c0da27886 100644 --- a/packages/lightning-mock/test/events/attempt-complete.test.ts +++ b/packages/lightning-mock/test/events/attempt-complete.test.ts @@ -1,7 +1,6 @@ import test from 'ava'; -import { ATTEMPT_COMPLETE } from '@openfn/ws-worker'; - import { join, setup, createAttempt } from '../util'; +import { ATTEMPT_COMPLETE } from '../../src/events'; let server; let client; diff --git a/packages/lightning-mock/test/events/attempt-start.test.ts b/packages/lightning-mock/test/events/attempt-start.test.ts index 19425842e..9e99d3c4e 100644 --- a/packages/lightning-mock/test/events/attempt-start.test.ts +++ b/packages/lightning-mock/test/events/attempt-start.test.ts @@ -1,7 +1,6 @@ import test from 'ava'; - import { join, setup, createAttempt } from '../util'; -import { ATTEMPT_START } from '@openfn/ws-worker'; +import { ATTEMPT_START } from '../../src/events'; let server; let client; diff --git a/packages/lightning-mock/test/events/log.test.ts b/packages/lightning-mock/test/events/log.test.ts index af4089e7b..b3a7ab3cb 100644 --- a/packages/lightning-mock/test/events/log.test.ts +++ b/packages/lightning-mock/test/events/log.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { ATTEMPT_LOG } from '@openfn/ws-worker'; +import { ATTEMPT_LOG } from '../../src/events'; import { join, setup, createAttempt } from '../util'; diff --git a/packages/lightning-mock/test/events/run-complete.test.ts b/packages/lightning-mock/test/events/run-complete.test.ts index 7c9bcdbef..3b4110696 100644 --- a/packages/lightning-mock/test/events/run-complete.test.ts +++ b/packages/lightning-mock/test/events/run-complete.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { RUN_COMPLETE } from '@openfn/ws-worker'; +import { RUN_COMPLETE } from '../../src/events'; import { join, setup, createAttempt } from '../util'; diff --git a/packages/lightning-mock/test/events/run-start.test.ts b/packages/lightning-mock/test/events/run-start.test.ts index 9ac20edb4..b478d35cf 100644 --- a/packages/lightning-mock/test/events/run-start.test.ts +++ b/packages/lightning-mock/test/events/run-start.test.ts @@ -1,6 +1,5 @@ import test from 'ava'; -import { RUN_START } from '@openfn/ws-worker'; - +import { RUN_START } from '../../src/events'; import { join, setup, createAttempt } from '../util'; let server; From 884d42f9a2e444a78906b102a78b9857548f61ec Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 17:30:03 +0000 Subject: [PATCH 18/45] lightning-mock: duplicate types from worker again We had a circular dependency which bigtime broke the build --- packages/lightning-mock/package.json | 1 - packages/lightning-mock/src/api-dev.ts | 8 ++- packages/lightning-mock/src/api-sockets.ts | 11 ++-- packages/lightning-mock/src/server.ts | 3 +- packages/lightning-mock/src/types.ts | 71 ++++++++++++++++++++++ 5 files changed, 82 insertions(+), 12 deletions(-) diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index a887c48ae..8b0bb45d1 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -31,7 +31,6 @@ "ws": "^8.14.1" }, "devDependencies": { - "@openfn/ws-worker": "workspace:^", "@types/koa": "^2.13.5", "@types/koa-bodyparser": "^4.3.10", "@types/koa-route": "^3.2.6", diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index 9c3af6f69..d3a32cbf6 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -9,8 +9,12 @@ import { ATTEMPT_COMPLETE } from './events'; import { ServerState } from './server'; -import type { Attempt, DevServer, LightningEvents } from './types'; -import type { AttemptCompletePayload } from '@openfn/ws-worker'; +import type { + AttemptCompletePayload, + Attempt, + DevServer, + LightningEvents, +} from './types'; type Api = { startAttempt(attemptId: string): void; diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 334d49ef8..dc648d618 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -1,9 +1,6 @@ import { WebSocketServer } from 'ws'; import createLogger, { Logger } from '@openfn/logger'; - -import type { ServerState } from './server'; - -import { extractAttemptId } from './util'; +import type { Server } from 'http'; import createPheonixMockSocketServer, { DevSocket, @@ -21,9 +18,9 @@ import { RUN_COMPLETE, RUN_START, } from './events'; -import type { Server } from 'http'; -import { stringify } from './util'; +import { extractAttemptId, stringify } from './util'; +import type { ServerState } from './server'; import type { AttemptStartPayload, AttemptStartReply, @@ -44,7 +41,7 @@ import type { RunCompleteReply, RunStartPayload, RunStartReply, -} from '@openfn/ws-worker'; +} from './types'; // dumb cloning id // just an idea for unit tests diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index f5440dedc..bd5f09f21 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -11,8 +11,7 @@ import createLogger, { import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; -import type { AttemptLogPayload } from '@openfn/ws-worker'; -import type { Attempt, DevServer } from './types'; +import type { AttemptLogPayload, Attempt, DevServer } from './types'; type RunId = string; type JobId = string; diff --git a/packages/lightning-mock/src/types.ts b/packages/lightning-mock/src/types.ts index b26bdbe4b..6eebf1ce3 100644 --- a/packages/lightning-mock/src/types.ts +++ b/packages/lightning-mock/src/types.ts @@ -62,3 +62,74 @@ export type DevServer = Koa & { startAttempt(id: string): any; waitForResult(attemptId: string): Promise; }; + +/** + * These are duplicated from the worker and subject to drift! + * We cannot import them directly because it creates a circular build dependency mock <-> worker + * We cannot declare an internal private types module because the generated dts will try to import from it + * + * The list of types is small enough right now that this is just about manageable + **/ +export type ExitReasonStrings = + | 'success' + | 'fail' + | 'crash' + | 'kill' + | 'cancel' + | 'exception'; + +export type ExitReason = { + reason: ExitReasonStrings; + error_message: string | null; + error_type: string | null; +}; + +export type ClaimPayload = { demand?: number }; +export type ClaimReply = { attempts: Array }; +export type ClaimAttempt = { id: string; token: string }; + +export type GetAttemptPayload = void; // no payload +export type GetAttemptReply = Attempt; + +export type GetCredentialPayload = { id: string }; +// credential in-line, no wrapper, arbitrary data +export type GetCredentialReply = {}; + +export type GetDataclipPayload = { id: string }; +export type GetDataClipReply = Uint8Array; // represents a json string Attempt + +export type AttemptStartPayload = void; // no payload +export type AttemptStartReply = {}; // no payload + +export type AttemptCompletePayload = ExitReason & { + final_dataclip_id?: string; // TODO this will be removed soon +}; +export type AttemptCompleteReply = undefined; + +export type AttemptLogPayload = { + message: Array; + timestamp: string; + attempt_id: string; + level?: string; + source?: string; // namespace + job_id?: string; + run_id?: string; +}; +export type AttemptLogReply = void; + +export type RunStartPayload = { + job_id: string; + run_id: string; + attempt_id?: string; + input_dataclip_id?: string; +}; +export type RunStartReply = void; + +export type RunCompletePayload = ExitReason & { + attempt_id?: string; + job_id: string; + run_id: string; + output_dataclip?: string; + output_dataclip_id?: string; +}; +export type RunCompleteReply = void; From 460b525bcc9057c249ef463f7a787ce39f1f46c2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 17:30:17 +0000 Subject: [PATCH 19/45] tidy typings --- packages/ws-worker/src/events.ts | 45 ------------------------------- packages/ws-worker/src/types.d.ts | 39 --------------------------- 2 files changed, 84 deletions(-) diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 390590b3e..e7aef8ddc 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -1,61 +1,16 @@ -import { Attempt, ExitReason } from './types'; - export const CLAIM = 'claim'; -export type ClaimPayload = { demand?: number }; -export type ClaimReply = { attempts: Array }; -export type ClaimAttempt = { id: string; token: string }; - export const GET_ATTEMPT = 'fetch:attempt'; -export type GetAttemptPayload = void; // no payload -export type GetAttemptReply = Attempt; export const GET_CREDENTIAL = 'fetch:credential'; -export type GetCredentialPayload = { id: string }; -// credential in-line, no wrapper, arbitrary data -export type GetCredentialReply = {}; export const GET_DATACLIP = 'fetch:dataclip'; -export type GetDataclipPayload = { id: string }; -export type GetDataClipReply = Uint8Array; // represents a json string Attempt export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp -export type AttemptStartPayload = void; // no payload -export type AttemptStartReply = {}; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats -export type AttemptCompletePayload = ExitReason & { - final_dataclip_id?: string; // TODO this will be removed soon -}; -export type AttemptCompleteReply = undefined; export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time -export type AttemptLogPayload = { - message: Array; - timestamp: string; - attempt_id: string; - level?: string; - source?: string; // namespace - job_id?: string; - run_id?: string; -}; -export type AttemptLogReply = void; - export const RUN_START = 'run:start'; -export type RunStartPayload = { - job_id: string; - run_id: string; - attempt_id?: string; - input_dataclip_id?: string; -}; -export type RunStartReply = void; export const RUN_COMPLETE = 'run:complete'; -export type RunCompletePayload = ExitReason & { - attempt_id?: string; - job_id: string; - run_id: string; - output_dataclip?: string; - output_dataclip_id?: string; -}; -export type RunCompleteReply = void; diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index 1d2b82f97..8552b54f9 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -103,13 +103,6 @@ type ReceiveHook = { ) => ReceiveHook; }; -// export declare class Socket extends PhxSocket { -// constructor(endpoint: string, options: { params: any }); -// onOpen(callback: () => void): void; -// connect(): void; -// channel(channelName: string, params: any): Channel; -// } - export interface Channel extends PhxChannel { // on: (event: string, fn: (evt: any) => void) => void; @@ -117,35 +110,3 @@ export interface Channel extends PhxChannel { push:

(event: string, payload?: P) => ReceiveHook; // join: () => ReceiveHook; } -// type RuntimeExecutionPlanID = string; - -// type JobEdge = { -// condition?: string; // Javascript expression (function body, not function) -// label?: string; -// }; - -// // TODO this type should later be imported from the runtime -// // Well, it's not quite the same is it, because eg credential can be a string -// type JobNode = { -// id?: string; - -// adaptor?: string; - -// expression?: string | Operation[]; // the code we actually want to execute. Can be a path. - -// configuration?: object | string; -// data?: State['data'] | string; // default state (globals) - -// next?: string | Record; -// }; - -// // Note that if the runtime itself is capable of calling an endpoint -// // To fetch credentials (and state?) just-in-time, then this is just a -// // Runtime Exeuction Plan, and we can import it. This is nicer tbh. -// export type ExecutionPlan = { -// id: string; // UUID for this plan - -// start?: JobNodeID; - -// plan: JobNode[]; // TODO this type should later be imported from the runtime -// }; From 063ba0b6b1bdf0daf9e5012da14457895c16c0b1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 17:30:56 +0000 Subject: [PATCH 20/45] package lock --- pnpm-lock.yaml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21705ef58..7aeceb6ab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -430,9 +430,6 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime - '@openfn/ws-worker': - specifier: workspace:^ - version: link:../ws-worker '@types/koa-logger': specifier: ^3.1.2 version: 3.1.2 @@ -1509,26 +1506,17 @@ packages: '@jridgewell/trace-mapping': 0.3.19 dev: true - /@jridgewell/resolve-uri@3.1.0: - resolution: {integrity: sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==} - engines: {node: '>=6.0.0'} - /@jridgewell/resolve-uri@3.1.1: resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} engines: {node: '>=6.0.0'} - dev: true /@jridgewell/set-array@1.1.2: resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} engines: {node: '>=6.0.0'} dev: true - /@jridgewell/sourcemap-codec@1.4.14: - resolution: {integrity: sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==} - /@jridgewell/sourcemap-codec@1.4.15: resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} - dev: true /@jridgewell/trace-mapping@0.3.19: resolution: {integrity: sha512-kf37QtfW+Hwx/buWGMPcR60iF9ziHa6r/CZJIHbmcm4+0qrXiVdxegAH0F6yddEVQ7zdkjcGCgCzUu+BcbhQxw==} @@ -1540,8 +1528,8 @@ packages: /@jridgewell/trace-mapping@0.3.9: resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} dependencies: - '@jridgewell/resolve-uri': 3.1.0 - '@jridgewell/sourcemap-codec': 1.4.14 + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 /@koa/router@12.0.0: resolution: {integrity: sha512-cnnxeKHXlt7XARJptflGURdJaO+ITpNkOHmQu7NHmCoRinPbyvFzce/EG/E8Zy81yQ1W9MoSdtklc3nyaDReUw==} From 4f52185e1567ddd62ffd42cea5358f0b3ba53cf6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 17:35:58 +0000 Subject: [PATCH 21/45] worker: fix typings --- packages/ws-worker/src/events.ts | 45 ++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index e7aef8ddc..390590b3e 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -1,16 +1,61 @@ +import { Attempt, ExitReason } from './types'; + export const CLAIM = 'claim'; +export type ClaimPayload = { demand?: number }; +export type ClaimReply = { attempts: Array }; +export type ClaimAttempt = { id: string; token: string }; + export const GET_ATTEMPT = 'fetch:attempt'; +export type GetAttemptPayload = void; // no payload +export type GetAttemptReply = Attempt; export const GET_CREDENTIAL = 'fetch:credential'; +export type GetCredentialPayload = { id: string }; +// credential in-line, no wrapper, arbitrary data +export type GetCredentialReply = {}; export const GET_DATACLIP = 'fetch:dataclip'; +export type GetDataclipPayload = { id: string }; +export type GetDataClipReply = Uint8Array; // represents a json string Attempt export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp +export type AttemptStartPayload = void; // no payload +export type AttemptStartReply = {}; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats +export type AttemptCompletePayload = ExitReason & { + final_dataclip_id?: string; // TODO this will be removed soon +}; +export type AttemptCompleteReply = undefined; export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time +export type AttemptLogPayload = { + message: Array; + timestamp: string; + attempt_id: string; + level?: string; + source?: string; // namespace + job_id?: string; + run_id?: string; +}; +export type AttemptLogReply = void; + export const RUN_START = 'run:start'; +export type RunStartPayload = { + job_id: string; + run_id: string; + attempt_id?: string; + input_dataclip_id?: string; +}; +export type RunStartReply = void; export const RUN_COMPLETE = 'run:complete'; +export type RunCompletePayload = ExitReason & { + attempt_id?: string; + job_id: string; + run_id: string; + output_dataclip?: string; + output_dataclip_id?: string; +}; +export type RunCompleteReply = void; From 20806eec7fe3997b0b5b5f088cab19822552be1a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 18:01:54 +0000 Subject: [PATCH 22/45] lightning-mock: comments --- packages/lightning-mock/src/events.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/lightning-mock/src/events.ts b/packages/lightning-mock/src/events.ts index 4b7eebd4a..36e389501 100644 --- a/packages/lightning-mock/src/events.ts +++ b/packages/lightning-mock/src/events.ts @@ -1,10 +1,3 @@ -/** - * These events are copied out of ws-worker - * There is a danger of them diverging - */ - -// TODO yeah for sure we need to remove these from here. Use the worker's types. - // new client connected export const CONNECT = 'socket:connect'; From 810397bb386c0ba28fc841173d4c77ae649439fe Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 15:05:12 +0000 Subject: [PATCH 23/45] worker: use real runtime in engine mock --- packages/ws-worker/src/mock/runtime-engine.ts | 272 +++++++++++------- .../test/mock/runtime-engine.test.ts | 60 ++-- 2 files changed, 216 insertions(+), 116 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 9e4380294..2cc5bd24f 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; import { EventEmitter } from 'node:events'; -import type { ExecutionPlan, JobNode } from '@openfn/runtime'; +import run, { ExecutionPlan, JobNode, NotifyEvents } from '@openfn/runtime'; import * as engine from '@openfn/engine-multi'; import mockResolvers from './resolvers'; @@ -26,6 +26,18 @@ export type WorkflowErrorEvent = { message: string; }; +// this is basically a fake adaptor +// these functions will be injected into scope +// maybe +// needs me to add the globals option to the runtime +// (which is fine) +const helpers = {}; + +// The mock runtime engine creates a fake engine interface +// around a real runtime engine +// Note that it does not dispatch runtime logs and only supports console.log +// This gives us real eventing in the worker tests +// TODO - even better would be to re-use the engine's event map or something async function createMock() { const activeWorkflows = {} as Record; const bus = new EventEmitter(); @@ -53,116 +65,176 @@ async function createMock() { listeners[planId] = events; }; - const executeJob = async ( - workflowId: string, - job: JobNode, - initialState = {}, - resolvers: engine.Resolvers = mockResolvers + const execute = async ( + xplan: ExecutionPlan, + options: { resolvers?: engine.Resolvers; throw?: boolean } = { + resolvers: mockResolvers, + } ) => { - const { id, expression, configuration, adaptor } = job; + const { id, jobs } = xplan; + activeWorkflows[id!] = true; - // If no expression or adaptor, this is (probably) a trigger node. - // Silently do nothing - if (!expression && !adaptor) { - return initialState; + // Call the crendtial callback, but don't do anything with it + for (const job of jobs) { + if (typeof job.configuration === 'string') { + job.configuration = await options.resolvers.credential?.( + job.configuration + ); + } } - const runId = crypto.randomUUID(); - - const jobId = id; - if (typeof configuration === 'string') { - // Fetch the credential but do nothing with it - // Maybe later we use it to assemble state - await resolvers.credential?.(configuration); - } + // TODO do I need a more sophisticated solution here? + const jobLogger = { + log: (...args) => { + dispatch('workflow-log', { + workflowId: id, + level: 'info', + json: true, + message: args, + }); + }, + }; - const info = (...message: any[]) => { - dispatch('workflow-log', { - workflowId, - message: message, - level: 'info', - time: (BigInt(Date.now()) * BigInt(1e3)).toString(), - name: 'mck', - }); + const opts = { + strict: false, + // logger? + jobLogger, + // linker? + ...options, + callbacks: { + notify: (name: NotifyEvents, payload: any) => { + // console.log(name, payload); + // TODO events need to be mapped into runtime engine events (noot runtime events) + dispatch(name, { + workflowId: id, + ...payload, // ? + }); + }, + }, }; + setTimeout(async () => { + dispatch('workflow-start', { workflowId: id }); - // Get the job details from lightning - // start instantly and emit as it goes - dispatch('job-start', { workflowId, jobId, runId }); - info('Running job ' + jobId); - let nextState = initialState; - - // @ts-ignore - if (expression?.startsWith?.('wait@')) { - const [_, delay] = (expression as string).split('@'); - nextState = initialState; - await new Promise((resolve) => { - setTimeout(() => resolve(), parseInt(delay)); - }); - } else { - // Try and parse the expression as JSON, in which case we use it as the final state - try { - // @ts-ignore - nextState = JSON.parse(expression); - // What does this look like? Should be a logger object - info('Parsing expression as JSON state'); - info(nextState); - } catch (e) { - // Do nothing, it's fine - nextState = initialState; - } - } + await run(xplan, undefined, opts); - dispatch('job-complete', { - workflowId, - jobId, - state: nextState, - runId, - next: [], // TODO hmm. I think we need to do better than this. - }); + delete activeWorkflows[id!]; + dispatch('workflow-complete', { workflowId: id }); + }, 1); - return nextState; + // Technically the engine should return an event emitter + // But as I don't think we use it, I'm happy to ignore this }; - // Start executing an ExecutionPlan - // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity - const execute = ( - xplan: ExecutionPlan, - options: { resolvers?: engine.Resolvers; throw?: boolean } = { - resolvers: mockResolvers, - } - ) => { - // This is just an easy way to test the options gets fed through to execute - // Also lets me test error handling! - if (options.throw) { - throw new Error('test error'); - } - - const { id, jobs, initialState } = xplan; - const workflowId = id; - activeWorkflows[id!] = true; - - // TODO do we want to load a globals dataclip from job.state here? - // This isn't supported right now - // We would need to use resolvers.dataclip if we wanted it - - setTimeout(() => { - dispatch('workflow-start', { workflowId }); - setTimeout(async () => { - let state = initialState || {}; - // Trivial job reducer in our mock - for (const job of jobs) { - state = await executeJob(id!, job, state, options.resolvers); - } - setTimeout(() => { - delete activeWorkflows[id!]; - dispatch('workflow-complete', { workflowId }); - // TODO on workflow complete we should maybe tidy the listeners? - // Doesn't really matter in the mock though - }, 1); - }, 1); - }, 1); - }; + // const executeJob = async ( + // workflowId: string, + // job: JobNode, + // initialState = {}, + // resolvers: engine.Resolvers = mockResolvers + // ) => { + // const { id, expression, configuration, adaptor } = job; + + // // If no expression or adaptor, this is (probably) a trigger node. + // // Silently do nothing + // if (!expression && !adaptor) { + // return initialState; + // } + + // const runId = crypto.randomUUID(); + + // const jobId = id; + // if (typeof configuration === 'string') { + // // Fetch the credential but do nothing with it + // // Maybe later we use it to assemble state + // await resolvers.credential?.(configuration); + // } + + // const info = (...message: any[]) => { + // dispatch('workflow-log', { + // workflowId, + // message: message, + // level: 'info', + // time: (BigInt(Date.now()) * BigInt(1e3)).toString(), + // name: 'mck', + // }); + // }; + + // // Get the job details from lightning + // // start instantly and emit as it goes + // dispatch('job-start', { workflowId, jobId, runId }); + // info('Running job ' + jobId); + // let nextState = initialState; + + // // @ts-ignore + // if (expression?.startsWith?.('wait@')) { + // const [_, delay] = (expression as string).split('@'); + // nextState = initialState; + // await new Promise((resolve) => { + // setTimeout(() => resolve(), parseInt(delay)); + // }); + // } else { + // // Try and parse the expression as JSON, in which case we use it as the final state + // try { + // // @ts-ignore + // nextState = JSON.parse(expression); + // // What does this look like? Should be a logger object + // info('Parsing expression as JSON state'); + // info(nextState); + // } catch (e) { + // // Do nothing, it's fine + // nextState = initialState; + // } + // } + + // dispatch('job-complete', { + // workflowId, + // jobId, + // state: nextState, + // runId, + // next: [], // TODO hmm. I think we need to do better than this. + // }); + + // return nextState; + // }; + + // // Start executing an ExecutionPlan + // // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity + // const execute = ( + // xplan: ExecutionPlan, + // options: { resolvers?: engine.Resolvers; throw?: boolean } = { + // resolvers: mockResolvers, + // } + // ) => { + // // This is just an easy way to test the options gets fed through to execute + // // Also lets me test error handling! + // if (options.throw) { + // throw new Error('test error'); + // } + + // const { id, jobs, initialState } = xplan; + // const workflowId = id; + // activeWorkflows[id!] = true; + + // // TODO do we want to load a globals dataclip from job.state here? + // // This isn't supported right now + // // We would need to use resolvers.dataclip if we wanted it + + // setTimeout(() => { + // dispatch('workflow-start', { workflowId }); + // setTimeout(async () => { + // let state = initialState || {}; + // // Trivial job reducer in our mock + // for (const job of jobs) { + // state = await executeJob(id!, job, state, options.resolvers); + // } + // setTimeout(() => { + // delete activeWorkflows[id!]; + // dispatch('workflow-complete', { workflowId }); + // // TODO on workflow complete we should maybe tidy the listeners? + // // Doesn't really matter in the mock though + // }, 1); + // }, 1); + // }, 1); + // }; // return a list of jobs in progress const getStatus = () => { diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index eb207ab4a..fabf88451 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -14,11 +14,17 @@ const sampleWorkflow = { { id: 'j1', adaptor: 'common@1.0.0', - expression: '{ "x": 10 }', + expression: 'export default [() => ({ x: 10 })]', }, ], } as ExecutionPlan; +// rethinking these tests, I think I want +// - fake adaptor functions, fn & wait +// - deeper testing on the engine events +// - fake compilation +// - do we do anything about adaptors? + test('getStatus() should should have no active workflows', async (t) => { const engine = await create(); const { active } = engine.getStatus(); @@ -85,7 +91,11 @@ test('mock should evaluate expressions as JSON', async (t) => { t.deepEqual(evt.state, { x: 10 }); }); -test('mock should wait if expression starts with @wait', async (t) => { +// well, maybe it shouldn't +// We should have real looking expressions +// albiet with no compilation +// we could fake compilation though +test.skip('mock should wait if expression starts with @wait', async (t) => { const engine = await create(); const wf = { id: 'w1', @@ -103,7 +113,8 @@ test('mock should wait if expression starts with @wait', async (t) => { t.true(end > 90); }); -test('mock should return initial state as result state', async (t) => { +// nope +test.skip('mock should return initial state as result state', async (t) => { const engine = await create(); const wf = { @@ -120,7 +131,8 @@ test('mock should return initial state as result state', async (t) => { t.deepEqual(evt.state, { y: 22 }); }); -test('mock prefers JSON state to initial state', async (t) => { +// nope +test.skip('mock prefers JSON state to initial state', async (t) => { const engine = await create(); const wf = { @@ -138,7 +150,8 @@ test('mock prefers JSON state to initial state', async (t) => { t.deepEqual(evt.state, { z: 33 }); }); -test('mock should dispatch log events when evaluating JSON', async (t) => { +// logs yes, json no +test.skip('mock should dispatch log events when evaluating JSON', async (t) => { const engine = await create(); const logs = []; @@ -153,7 +166,8 @@ test('mock should dispatch log events when evaluating JSON', async (t) => { t.deepEqual(logs[1].message, ['Parsing expression as JSON state']); }); -test('mock should throw if the magic option is passed', async (t) => { +// nope, the engine should not throw at all +test.skip('mock should throw if the magic option is passed', async (t) => { const engine = await create(); const logs = []; @@ -198,34 +212,45 @@ test('listen to events', async (t) => { 'workflow-complete': false, }; - engine.listen(sampleWorkflow.id, { + const wf = { + id: 'wibble', + jobs: [ + { + id: 'j1', + adaptor: 'common@1.0.0', + expression: 'export default [() => { console.log("x"); }]', + }, + ], + } as ExecutionPlan; + + engine.listen(wf.id, { 'job-start': ({ workflowId, jobId }) => { called['job-start'] = true; - t.is(workflowId, sampleWorkflow.id); - t.is(jobId, sampleWorkflow.jobs[0].id); + t.is(workflowId, wf.id); + t.is(jobId, wf.jobs[0].id); }, 'job-complete': ({ workflowId, jobId }) => { called['job-complete'] = true; - t.is(workflowId, sampleWorkflow.id); - t.is(jobId, sampleWorkflow.jobs[0].id); + t.is(workflowId, wf.id); + t.is(jobId, wf.jobs[0].id); // TODO includes state? }, 'workflow-log': ({ workflowId, message }) => { called['workflow-log'] = true; - t.is(workflowId, sampleWorkflow.id); + t.is(workflowId, wf.id); t.truthy(message); }, 'workflow-start': ({ workflowId }) => { called['workflow-start'] = true; - t.is(workflowId, sampleWorkflow.id); + t.is(workflowId, wf.id); }, 'workflow-complete': ({ workflowId }) => { called['workflow-complete'] = true; - t.is(workflowId, sampleWorkflow.id); + t.is(workflowId, wf.id); }, }); - engine.execute(sampleWorkflow); + engine.execute(wf); await waitForEvent(engine, 'workflow-complete'); t.assert(Object.values(called).every((v) => v === true)); }); @@ -244,12 +269,13 @@ test('only listen to events for the correct workflow', async (t) => { t.pass(); }); -test('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { +test.skip('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { const workflow = { id: 'w1', jobs: [ { id: 'j1', + expression: 'export default [() => console.log("x"); )]', }, ], } as ExecutionPlan; @@ -279,5 +305,7 @@ test('do nothing for a job if no expression and adaptor (trigger node)', async ( engine.execute(workflow); await waitForEvent(engine, 'workflow-complete'); + console.log(); + t.false(didCallEvent); }); From f7f504820fd3b047cba0650ad246d82854e49577 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 15:26:39 +0000 Subject: [PATCH 24/45] worker: fix tests with new mock --- packages/ws-worker/src/mock/runtime-engine.ts | 12 +++- packages/ws-worker/test/api/execute.test.ts | 14 +++-- packages/ws-worker/test/lightning.test.ts | 57 +++++++++++-------- 3 files changed, 52 insertions(+), 31 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 2cc5bd24f..65df0398f 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -91,6 +91,7 @@ async function createMock() { level: 'info', json: true, message: args, + time: Date.now(), }); }, }; @@ -115,7 +116,16 @@ async function createMock() { setTimeout(async () => { dispatch('workflow-start', { workflowId: id }); - await run(xplan, undefined, opts); + try { + await run(xplan, undefined, opts); + } catch (e: any) { + // TODO I have no test on this + dispatch('workflow-error', { + workflowId: id, + type: e.name, + message: e.message, + }); + } delete activeWorkflows[id!]; dispatch('workflow-complete', { workflowId: id }); diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index d6af0e965..a74832edb 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -407,7 +407,8 @@ test('execute should pass the final result to onFinish', async (t) => { id: 'a', jobs: [ { - expression: JSON.stringify({ done: true }), + // TODO use new fn helper + expression: 'export default [() => ({ done: true })]', }, ], }; @@ -431,7 +432,8 @@ test('execute should return a context object', async (t) => { id: 'a', jobs: [ { - expression: JSON.stringify({ done: true }), + // TODO use new fn helper + expression: 'export default [() => ({ done: true })]', }, ], }; @@ -476,7 +478,8 @@ test('execute should lazy-load a credential', async (t) => { jobs: [ { configuration: 'abc', - expression: JSON.stringify({ done: true }), + // TODO use new fn helper + expression: 'export default [() => ({ done: true })]', }, ], }; @@ -511,7 +514,8 @@ test('execute should lazy-load initial state', async (t) => { initialState: 'abc', jobs: [ { - expression: JSON.stringify({ done: true }), + // TODO use new fn helper + expression: 'export default [() => ({ done: true })]', }, ], }; @@ -558,8 +562,8 @@ test('execute should call all events on the socket', async (t) => { { id: 'trigger', configuration: 'a', - expression: 'fn(a => a)', adaptor: '@openfn/language-common@1.0.0', + expression: 'export default [() => console.log("x")]', }, ], }; diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts index faf004527..bac5b9754 100644 --- a/packages/ws-worker/test/lightning.test.ts +++ b/packages/ws-worker/test/lightning.test.ts @@ -30,13 +30,18 @@ test.before(async () => { let rollingAttemptId = 0; +// simulate an fn operation without compilation +// TODO even better to mock cmompilation tbh +const fn = (expression: string) => + `const fn = (f) => (s) => f(s); export default [${expression}]`; + const getAttempt = (ext = {}, jobs?: any) => ({ id: `a${++rollingAttemptId}`, jobs: jobs || [ { id: 'j', adaptor: '@openfn/language-common@1.0.0', - body: JSON.stringify({ answer: 42 }), + body: fn('() => ({ answer: 42 })'), }, ], ...ext, @@ -51,7 +56,7 @@ test.serial( id: 'attempt-1', jobs: [ { - body: JSON.stringify({ count: 122 }), + body: fn('() => ({ count: 122 })'), }, ], }; @@ -69,7 +74,7 @@ test.serial( test.serial('should run an attempt which returns intial state', async (t) => { return new Promise((done) => { lng.addDataclip('x', { - route: 66, + data: 66, }); const attempt = { @@ -77,13 +82,13 @@ test.serial('should run an attempt which returns intial state', async (t) => { dataclip_id: 'x', jobs: [ { - body: 'whatever', + body: fn('(s) => s'), }, ], }; lng.waitForResult(attempt.id).then((result) => { - t.deepEqual(result, { route: 66 }); + t.deepEqual(result, { data: 66 }); done(); }); @@ -173,17 +178,17 @@ test.serial( id: 'some-job', credential_id: 'a', adaptor: '@openfn/language-common@1.0.0', - body: JSON.stringify({ answer: 42 }), + body: fn('() => ({ answer: 42 })'), }, ]); let didCallEvent = false; - lng.onSocketEvent(e.GET_CREDENTIAL, attempt.id, ({ payload }) => { + lng.onSocketEvent(e.GET_CREDENTIAL, attempt.id, () => { // again there's no way to check the right credential was returned didCallEvent = true; }); - lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, () => { t.true(didCallEvent); done(); }); @@ -268,11 +273,15 @@ test.serial( `events: lightning should receive a ${e.ATTEMPT_LOG} event`, (t) => { return new Promise((done) => { - const attempt = getAttempt(); - - let didCallEvent = false; + const attempt = { + id: 'attempt-1', + jobs: [ + { + body: fn('(s) => { console.log("x"); return s }'), + }, + ], + }; - // The mock runtime will put out a default log lng.onSocketEvent(e.ATTEMPT_LOG, attempt.id, ({ payload }) => { const log = payload; @@ -280,13 +289,10 @@ test.serial( t.truthy(log.attempt_id); t.truthy(log.run_id); t.truthy(log.message); - t.assert(log.message[0].startsWith('Running job')); - - didCallEvent = true; + t.deepEqual(log.message, ['x']); }); lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { - t.true(didCallEvent); done(); }); @@ -300,13 +306,13 @@ test.serial( test.serial.skip(`events: logs should have increasing timestamps`, (t) => { return new Promise((done) => { const attempt = getAttempt({}, [ - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, - { body: '{ x: 1 }', adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, ]); const history: bigint[] = []; @@ -372,7 +378,7 @@ test('should register and de-register attempts to the server', async (t) => { id: 'attempt-1', jobs: [ { - body: JSON.stringify({ count: 122 }), + body: fn('() => ({ count: 122 })'), }, ], }; @@ -398,7 +404,8 @@ test('should register and de-register attempts to the server', async (t) => { // TODO this is a server test // What I am testing here is that the first job completes // before the second job starts -test('should not claim while at capacity', async (t) => { +// TODO add wait helper +test.skip('should not claim while at capacity', async (t) => { return new Promise((done) => { const attempt1 = { id: 'attempt-1', From acf2f7cdbcb127404fce92ad4ac64e083981637a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 15:27:53 +0000 Subject: [PATCH 25/45] tidyup --- packages/ws-worker/src/mock/runtime-engine.ts | 119 +----------------- packages/ws-worker/test/api/execute.test.ts | 3 +- 2 files changed, 4 insertions(+), 118 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 65df0398f..ca46b8a13 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,6 +1,5 @@ -import crypto from 'node:crypto'; import { EventEmitter } from 'node:events'; -import run, { ExecutionPlan, JobNode, NotifyEvents } from '@openfn/runtime'; +import run, { ExecutionPlan, NotifyEvents } from '@openfn/runtime'; import * as engine from '@openfn/engine-multi'; import mockResolvers from './resolvers'; @@ -74,9 +73,9 @@ async function createMock() { const { id, jobs } = xplan; activeWorkflows[id!] = true; - // Call the crendtial callback, but don't do anything with it for (const job of jobs) { if (typeof job.configuration === 'string') { + // Call the crendtial callback, but don't do anything with it job.configuration = await options.resolvers.credential?.( job.configuration ); @@ -98,13 +97,10 @@ async function createMock() { const opts = { strict: false, - // logger? jobLogger, - // linker? ...options, callbacks: { notify: (name: NotifyEvents, payload: any) => { - // console.log(name, payload); // TODO events need to be mapped into runtime engine events (noot runtime events) dispatch(name, { workflowId: id, @@ -135,117 +131,6 @@ async function createMock() { // But as I don't think we use it, I'm happy to ignore this }; - // const executeJob = async ( - // workflowId: string, - // job: JobNode, - // initialState = {}, - // resolvers: engine.Resolvers = mockResolvers - // ) => { - // const { id, expression, configuration, adaptor } = job; - - // // If no expression or adaptor, this is (probably) a trigger node. - // // Silently do nothing - // if (!expression && !adaptor) { - // return initialState; - // } - - // const runId = crypto.randomUUID(); - - // const jobId = id; - // if (typeof configuration === 'string') { - // // Fetch the credential but do nothing with it - // // Maybe later we use it to assemble state - // await resolvers.credential?.(configuration); - // } - - // const info = (...message: any[]) => { - // dispatch('workflow-log', { - // workflowId, - // message: message, - // level: 'info', - // time: (BigInt(Date.now()) * BigInt(1e3)).toString(), - // name: 'mck', - // }); - // }; - - // // Get the job details from lightning - // // start instantly and emit as it goes - // dispatch('job-start', { workflowId, jobId, runId }); - // info('Running job ' + jobId); - // let nextState = initialState; - - // // @ts-ignore - // if (expression?.startsWith?.('wait@')) { - // const [_, delay] = (expression as string).split('@'); - // nextState = initialState; - // await new Promise((resolve) => { - // setTimeout(() => resolve(), parseInt(delay)); - // }); - // } else { - // // Try and parse the expression as JSON, in which case we use it as the final state - // try { - // // @ts-ignore - // nextState = JSON.parse(expression); - // // What does this look like? Should be a logger object - // info('Parsing expression as JSON state'); - // info(nextState); - // } catch (e) { - // // Do nothing, it's fine - // nextState = initialState; - // } - // } - - // dispatch('job-complete', { - // workflowId, - // jobId, - // state: nextState, - // runId, - // next: [], // TODO hmm. I think we need to do better than this. - // }); - - // return nextState; - // }; - - // // Start executing an ExecutionPlan - // // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity - // const execute = ( - // xplan: ExecutionPlan, - // options: { resolvers?: engine.Resolvers; throw?: boolean } = { - // resolvers: mockResolvers, - // } - // ) => { - // // This is just an easy way to test the options gets fed through to execute - // // Also lets me test error handling! - // if (options.throw) { - // throw new Error('test error'); - // } - - // const { id, jobs, initialState } = xplan; - // const workflowId = id; - // activeWorkflows[id!] = true; - - // // TODO do we want to load a globals dataclip from job.state here? - // // This isn't supported right now - // // We would need to use resolvers.dataclip if we wanted it - - // setTimeout(() => { - // dispatch('workflow-start', { workflowId }); - // setTimeout(async () => { - // let state = initialState || {}; - // // Trivial job reducer in our mock - // for (const job of jobs) { - // state = await executeJob(id!, job, state, options.resolvers); - // } - // setTimeout(() => { - // delete activeWorkflows[id!]; - // dispatch('workflow-complete', { workflowId }); - // // TODO on workflow complete we should maybe tidy the listeners? - // // Doesn't really matter in the mock though - // }, 1); - // }, 1); - // }, 1); - // }; - // return a list of jobs in progress const getStatus = () => { return { diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index a74832edb..a091bb68e 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -24,7 +24,8 @@ import { import createMockRTE from '../../src/mock/runtime-engine'; import { mockChannel } from '../../src/mock/sockets'; import { stringify, createAttemptState } from '../../src/util'; -import { ExecutionPlan } from '@openfn/runtime'; + +import type { ExecutionPlan } from '@openfn/runtime'; import type { AttemptState } from '../../src/types'; const enc = new TextEncoder(); From 1ea8e6e707a033fbd592d75c125ce3b212e5e2da Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 16:37:50 +0000 Subject: [PATCH 26/45] worker: update tests to use new helpers --- packages/ws-worker/src/mock/runtime-engine.ts | 15 +- packages/ws-worker/test/api/execute.test.ts | 17 +-- packages/ws-worker/test/lightning.test.ts | 36 ++--- .../test/mock/runtime-engine.test.ts | 132 ++++-------------- 4 files changed, 58 insertions(+), 142 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index ca46b8a13..06f1e374e 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -27,10 +27,11 @@ export type WorkflowErrorEvent = { // this is basically a fake adaptor // these functions will be injected into scope -// maybe -// needs me to add the globals option to the runtime -// (which is fine) -const helpers = {}; +const helpers = { + fn: (f: Function) => (s: any) => f(s), + wait: (duration: number) => (s: any) => + new Promise((resolve) => setTimeout(resolve, duration)), +}; // The mock runtime engine creates a fake engine interface // around a real runtime engine @@ -80,6 +81,11 @@ async function createMock() { job.configuration ); } + + // Fake compilation + if (job.expression && !job.expression.match(/export default \[/)) { + job.expression = `export default [${job.expression}];`; + } } // TODO do I need a more sophisticated solution here? @@ -99,6 +105,7 @@ async function createMock() { strict: false, jobLogger, ...options, + globals: helpers, callbacks: { notify: (name: NotifyEvents, payload: any) => { // TODO events need to be mapped into runtime engine events (noot runtime events) diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index a091bb68e..2b7c58e46 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -408,8 +408,7 @@ test('execute should pass the final result to onFinish', async (t) => { id: 'a', jobs: [ { - // TODO use new fn helper - expression: 'export default [() => ({ done: true })]', + expression: 'fn(() => ({ done: true }))', }, ], }; @@ -433,8 +432,7 @@ test('execute should return a context object', async (t) => { id: 'a', jobs: [ { - // TODO use new fn helper - expression: 'export default [() => ({ done: true })]', + expression: 'fn(() => ({ done: true }))', }, ], }; @@ -479,8 +477,7 @@ test('execute should lazy-load a credential', async (t) => { jobs: [ { configuration: 'abc', - // TODO use new fn helper - expression: 'export default [() => ({ done: true })]', + expression: 'fn(() => ({ done: true }))', }, ], }; @@ -515,8 +512,7 @@ test('execute should lazy-load initial state', async (t) => { initialState: 'abc', jobs: [ { - // TODO use new fn helper - expression: 'export default [() => ({ done: true })]', + expression: 'fn(() => ({ done: true }))', }, ], }; @@ -550,7 +546,7 @@ test('execute should call all events on the socket', async (t) => { // GET_DATACLIP, // TODO not really implemented properly yet ATTEMPT_START, RUN_START, - ATTEMPT_LOG, // This won't log with the mock logger + ATTEMPT_LOG, RUN_COMPLETE, ATTEMPT_COMPLETE, ]; @@ -564,7 +560,7 @@ test('execute should call all events on the socket', async (t) => { id: 'trigger', configuration: 'a', adaptor: '@openfn/language-common@1.0.0', - expression: 'export default [() => console.log("x")]', + expression: 'fn(() => console.log("x"))', }, ], }; @@ -573,7 +569,6 @@ test('execute should call all events on the socket', async (t) => { return new Promise((done) => { execute(channel, engine, logger, plan, options, (result) => { - // console.log(events); // Check that events were passed to the socket // This is deliberately crude t.assert(allEvents.every((e) => events[e])); diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts index bac5b9754..3fe37558b 100644 --- a/packages/ws-worker/test/lightning.test.ts +++ b/packages/ws-worker/test/lightning.test.ts @@ -30,18 +30,13 @@ test.before(async () => { let rollingAttemptId = 0; -// simulate an fn operation without compilation -// TODO even better to mock cmompilation tbh -const fn = (expression: string) => - `const fn = (f) => (s) => f(s); export default [${expression}]`; - const getAttempt = (ext = {}, jobs?: any) => ({ id: `a${++rollingAttemptId}`, jobs: jobs || [ { id: 'j', adaptor: '@openfn/language-common@1.0.0', - body: fn('() => ({ answer: 42 })'), + body: 'fn(() => ({ answer: 42 }))', }, ], ...ext, @@ -56,7 +51,7 @@ test.serial( id: 'attempt-1', jobs: [ { - body: fn('() => ({ count: 122 })'), + body: 'fn(() => ({ count: 122 }))', }, ], }; @@ -82,7 +77,7 @@ test.serial('should run an attempt which returns intial state', async (t) => { dataclip_id: 'x', jobs: [ { - body: fn('(s) => s'), + body: 'fn((s) => s)', }, ], }; @@ -178,7 +173,7 @@ test.serial( id: 'some-job', credential_id: 'a', adaptor: '@openfn/language-common@1.0.0', - body: fn('() => ({ answer: 42 })'), + body: 'fn(() => ({ answer: 42 }))', }, ]); @@ -277,7 +272,7 @@ test.serial( id: 'attempt-1', jobs: [ { - body: fn('(s) => { console.log("x"); return s }'), + body: 'fn((s) => { console.log("x"); return s })', }, ], }; @@ -306,13 +301,14 @@ test.serial( test.serial.skip(`events: logs should have increasing timestamps`, (t) => { return new Promise((done) => { const attempt = getAttempt({}, [ - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, - { body: fn('() => ({ data: 1 })'), adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, + { body: 'fn(() => ({ data: 1 }))', adaptor: 'common' }, ]); const history: bigint[] = []; @@ -378,7 +374,7 @@ test('should register and de-register attempts to the server', async (t) => { id: 'attempt-1', jobs: [ { - body: fn('() => ({ count: 122 })'), + body: 'fn(() => ({ count: 122 }))', }, ], }; @@ -411,7 +407,7 @@ test.skip('should not claim while at capacity', async (t) => { id: 'attempt-1', jobs: [ { - body: 'wait@500', + body: 'wait(500)', }, ], }; @@ -453,5 +449,3 @@ test.skip('should not claim while at capacity', async (t) => { // hmm, i don't even think I can test this in the mock runtime test.skip('should pass the right dataclip when running in parallel', () => {}); - -test.todo(`should run multiple attempts`); diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index fabf88451..14dc2b843 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -14,27 +14,24 @@ const sampleWorkflow = { { id: 'j1', adaptor: 'common@1.0.0', - expression: 'export default [() => ({ x: 10 })]', + expression: 'fn(() => ({ data: { x: 10 } }))', }, ], } as ExecutionPlan; -// rethinking these tests, I think I want -// - fake adaptor functions, fn & wait -// - deeper testing on the engine events -// - fake compilation -// - do we do anything about adaptors? +let engine; + +test.before(async () => { + engine = await create(); +}); test('getStatus() should should have no active workflows', async (t) => { - const engine = await create(); const { active } = engine.getStatus(); t.is(active, 0); }); test('Dispatch start events for a new workflow', async (t) => { - const engine = await create(); - engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'workflow-start'); t.truthy(evt); @@ -42,7 +39,6 @@ test('Dispatch start events for a new workflow', async (t) => { }); test('getStatus should report one active workflow', async (t) => { - const engine = await create(); engine.execute(sampleWorkflow); const { active } = engine.getStatus(); @@ -51,8 +47,6 @@ test('getStatus should report one active workflow', async (t) => { }); test('Dispatch complete events when a workflow completes', async (t) => { - const engine = await create(); - engine.execute(sampleWorkflow); const evt = await waitForEvent( engine, @@ -63,8 +57,6 @@ test('Dispatch complete events when a workflow completes', async (t) => { }); test('Dispatch start events for a job', async (t) => { - const engine = await create(); - engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'job-start'); t.truthy(evt); @@ -73,114 +65,51 @@ test('Dispatch start events for a job', async (t) => { }); test('Dispatch complete events for a job', async (t) => { - const engine = await create(); - engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'job-complete'); t.truthy(evt); t.is(evt.workflowId, 'w1'); t.is(evt.jobId, 'j1'); - t.truthy(evt.state); -}); - -test('mock should evaluate expressions as JSON', async (t) => { - const engine = await create(); - - engine.execute(sampleWorkflow); - const evt = await waitForEvent(engine, 'job-complete'); - t.deepEqual(evt.state, { x: 10 }); + t.deepEqual(evt.state, { data: { x: 10 } }); }); -// well, maybe it shouldn't -// We should have real looking expressions -// albiet with no compilation -// we could fake compilation though -test.skip('mock should wait if expression starts with @wait', async (t) => { - const engine = await create(); +test('Dispatch error event for a crash', async (t) => { const wf = { - id: 'w1', + id: 'xyz', jobs: [ { id: 'j1', - expression: 'wait@100', + adaptor: 'common@1.0.0', + expression: 'fn(() => ( @~!"@£!4 )', }, ], } as ExecutionPlan; - engine.execute(wf); - const start = Date.now(); - const evt = await waitForEvent(engine, 'workflow-complete'); - const end = Date.now() - start; - t.true(end > 90); -}); -// nope -test.skip('mock should return initial state as result state', async (t) => { - const engine = await create(); - - const wf = { - initialState: { y: 22 }, - jobs: [ - { - adaptor: 'common@1.0.0', - }, - ], - }; engine.execute(wf); + const evt = await waitForEvent(engine, 'workflow-error'); - const evt = await waitForEvent(engine, 'job-complete'); - t.deepEqual(evt.state, { y: 22 }); + t.is(evt.workflowId, 'xyz'); + t.is(evt.type, 'RuntimeCrash'); + t.regex(evt.message, /invalid or unexpected token/i); }); -// nope -test.skip('mock prefers JSON state to initial state', async (t) => { - const engine = await create(); - +test('wait function', async (t) => { const wf = { - initialState: { y: 22 }, + id: 'w1', jobs: [ { - adaptor: 'common@1.0.0', - expression: '{ "z": 33 }', + id: 'j1', + expression: 'wait(100)', }, ], - }; + } as ExecutionPlan; engine.execute(wf); + const start = Date.now(); - const evt = await waitForEvent(engine, 'job-complete'); - t.deepEqual(evt.state, { z: 33 }); -}); - -// logs yes, json no -test.skip('mock should dispatch log events when evaluating JSON', async (t) => { - const engine = await create(); - - const logs = []; - engine.on('workflow-log', (l) => { - logs.push(l); - }); - - engine.execute(sampleWorkflow); - await waitForEvent(engine, 'workflow-complete'); - - t.deepEqual(logs[0].message, ['Running job j1']); - t.deepEqual(logs[1].message, ['Parsing expression as JSON state']); -}); - -// nope, the engine should not throw at all -test.skip('mock should throw if the magic option is passed', async (t) => { - const engine = await create(); + await waitForEvent(engine, 'workflow-complete'); - const logs = []; - engine.on('workflow-log', (l) => { - logs.push(l); - }); - - await t.throwsAsync( - async () => engine.execute(sampleWorkflow, { throw: true }), - { - message: 'test error', - } - ); + const end = Date.now() - start; + t.true(end > 90); }); test('resolve credential before job-start if credential is a string', async (t) => { @@ -193,7 +122,6 @@ test('resolve credential before job-start if credential is a string', async (t) return {}; }; - const engine = await create(); // @ts-ignore engine.execute(wf, { resolvers: { credential } }); @@ -202,8 +130,6 @@ test('resolve credential before job-start if credential is a string', async (t) }); test('listen to events', async (t) => { - const engine = await create(); - const called = { 'job-start': false, 'job-complete': false, @@ -256,8 +182,6 @@ test('listen to events', async (t) => { }); test('only listen to events for the correct workflow', async (t) => { - const engine = await create(); - engine.listen('bobby mcgee', { 'workflow-start': ({ workflowId }) => { throw new Error('should not have called this!!'); @@ -269,19 +193,17 @@ test('only listen to events for the correct workflow', async (t) => { t.pass(); }); -test.skip('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { +test('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { const workflow = { id: 'w1', jobs: [ { id: 'j1', - expression: 'export default [() => console.log("x"); )]', + adaptor: '@openfn/language-common@1.0.0', }, ], } as ExecutionPlan; - const engine = await create(); - let didCallEvent = false; engine.listen(workflow.id, { @@ -305,7 +227,5 @@ test.skip('do nothing for a job if no expression and adaptor (trigger node)', as engine.execute(workflow); await waitForEvent(engine, 'workflow-complete'); - console.log(); - t.false(didCallEvent); }); From c2aa7ce8e4a72c5a5d1ec57b77d5d562237ee7a5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 17:12:56 +0000 Subject: [PATCH 27/45] lightning-mock: allow to unsubscribe from listeners --- packages/lightning-mock/src/api-dev.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index d3a32cbf6..6ddf7260e 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -111,16 +111,18 @@ const setupDevAPI = ( attemptId: string, fn: (evt: any) => void, once = true - ) => { + ): (() => void) => { + const unsubscribe = () => state.events.removeListener(event, handler); function handler(e: any) { if (e.attemptId && e.attemptId === attemptId) { if (once) { - state.events.removeListener(event, handler); + unsubscribe(); } fn(e); } } state.events.addListener(event, handler); + return unsubscribe; }; }; From 47f491add64befee605b079cb223cb7767e13817 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 17:18:06 +0000 Subject: [PATCH 28/45] worker: epic parallelisation test --- packages/ws-worker/test/lightning.test.ts | 84 ++++++++++++++++++++++- 1 file changed, 82 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts index 3fe37558b..72cf22167 100644 --- a/packages/ws-worker/test/lightning.test.ts +++ b/packages/ws-worker/test/lightning.test.ts @@ -447,5 +447,85 @@ test.skip('should not claim while at capacity', async (t) => { }); }); -// hmm, i don't even think I can test this in the mock runtime -test.skip('should pass the right dataclip when running in parallel', () => {}); +test('should pass the right dataclip when running in parallel', (t) => { + return new Promise((done) => { + const job = (id: string, next?: string) => ({ + id, + body: `fn((s) => { s.data.${id} = true; return s; })`, + }); + + const edge = (from: string, to: string) => ({ + id: `${from}-${to}`, + source_job_id: from, + target_job_id: to, + }); + + const outputDataclipIds = {}; + const inputDataclipIds = {}; + const outputs = {}; + const a = { + id: 'a', + body: 'fn(() => ({ data: { a: true } }))', + next: { j: true, k: true }, + }; + + const j = job('j', 'x'); + const k = job('k', 'y'); + const x = job('x'); + const y = job('y'); + + const attempt = { + id: 'p1', + jobs: [a, j, k, x, y], + edges: [edge('a', 'j'), edge('a', 'k'), edge('j', 'x'), edge('k', 'y')], + }; + + // Save all the input dataclip ids for each job + const unsub2 = lng.onSocketEvent( + e.RUN_START, + attempt.id, + ({ payload }) => { + inputDataclipIds[payload.job_id] = payload.input_dataclip_id; + }, + false + ); + + // Save all the output dataclips & ids for each job + const unsub1 = lng.onSocketEvent( + e.RUN_COMPLETE, + attempt.id, + ({ payload }) => { + outputDataclipIds[payload.job_id] = payload.output_dataclip_id; + outputs[payload.job_id] = JSON.parse(payload.output_dataclip); + }, + false + ); + + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + unsub1(); + unsub2(); + + // Now check everything was correct + + // Job a we don't really care about, but check the output anyway + t.deepEqual(outputs.a.data, { a: true }); + + // a feeds in to j and k + t.deepEqual(inputDataclipIds.j, outputDataclipIds.a); + t.deepEqual(inputDataclipIds.k, outputDataclipIds.a); + + // j feeds into x + t.deepEqual(inputDataclipIds.x, outputDataclipIds.j); + + // k feeds into y + t.deepEqual(inputDataclipIds.y, outputDataclipIds.k); + + // x and y should have divergent states + t.deepEqual(outputs.x.data, { a: true, j: true, x: true }); + t.deepEqual(outputs.y.data, { a: true, k: true, y: true }); + done(); + }); + + lng.enqueueAttempt(attempt); + }); +}); From 431b43aaf9003825634dda321adc16f045a1f38c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 17:35:28 +0000 Subject: [PATCH 29/45] worker: typings and tidies --- packages/ws-worker/src/mock/runtime-engine.ts | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 06f1e374e..bbf5324e6 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import run, { ExecutionPlan, NotifyEvents } from '@openfn/runtime'; +import run, { ExecutionPlan } from '@openfn/runtime'; import * as engine from '@openfn/engine-multi'; import mockResolvers from './resolvers'; @@ -30,7 +30,7 @@ export type WorkflowErrorEvent = { const helpers = { fn: (f: Function) => (s: any) => f(s), wait: (duration: number) => (s: any) => - new Promise((resolve) => setTimeout(resolve, duration)), + new Promise((resolve) => setTimeout(() => resolve(s), duration)), }; // The mock runtime engine creates a fake engine interface @@ -77,20 +77,23 @@ async function createMock() { for (const job of jobs) { if (typeof job.configuration === 'string') { // Call the crendtial callback, but don't do anything with it - job.configuration = await options.resolvers.credential?.( + job.configuration = await options.resolvers?.credential?.( job.configuration ); } // Fake compilation - if (job.expression && !job.expression.match(/export default \[/)) { + if ( + typeof job.expression === 'string' && + !(job.expression as string).match(/export default \[/) + ) { job.expression = `export default [${job.expression}];`; } } // TODO do I need a more sophisticated solution here? const jobLogger = { - log: (...args) => { + log: (...args: any[]) => { dispatch('workflow-log', { workflowId: id, level: 'info', @@ -107,11 +110,10 @@ async function createMock() { ...options, globals: helpers, callbacks: { - notify: (name: NotifyEvents, payload: any) => { - // TODO events need to be mapped into runtime engine events (noot runtime events) + notify: (name: any, payload: any) => { dispatch(name, { workflowId: id, - ...payload, // ? + ...payload, }); }, }, @@ -120,9 +122,8 @@ async function createMock() { dispatch('workflow-start', { workflowId: id }); try { - await run(xplan, undefined, opts); + await run(xplan, undefined, opts as any); } catch (e: any) { - // TODO I have no test on this dispatch('workflow-error', { workflowId: id, type: e.name, From 40ffc226edecb728322fdd0c76114a56147625d8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Nov 2023 17:39:39 +0000 Subject: [PATCH 30/45] runtime: changeset --- .changeset/lovely-dodos-guess.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/lovely-dodos-guess.md diff --git a/.changeset/lovely-dodos-guess.md b/.changeset/lovely-dodos-guess.md new file mode 100644 index 000000000..cf75f4a47 --- /dev/null +++ b/.changeset/lovely-dodos-guess.md @@ -0,0 +1,5 @@ +--- +'@openfn/runtime': minor +--- + +Allow globals to be passed into the execution environment From bed51546c33569ff911ac14a555ee5149d240a1a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 20 Nov 2023 17:50:43 +0000 Subject: [PATCH 31/45] deploy: move enabled flag to an Edge --- packages/deploy/src/stateTransform.ts | 3 +-- packages/deploy/src/types.ts | 3 ++- packages/deploy/test/fixtures.ts | 24 ++++++++++----------- packages/deploy/test/stateTransform.test.ts | 12 ++++------- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index fe7b9dd6f..c7a0424d1 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -33,7 +33,6 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: specJob.body, - enabled: pickValue(specJob, stateJob || {}, 'enabled', true), }, ]; } @@ -50,7 +49,6 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: specJob.body, - enabled: pickValue(specJob, stateJob, 'enabled', true), }, ]; } @@ -146,6 +144,7 @@ function mergeEdges( id, condition: specEdge.condition ?? null, target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '', + enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true), }, { source_job_id: jobs[specEdge.source_job ?? -1]?.id, diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index cd9b2c709..992ff7774 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -2,7 +2,6 @@ export type Job = { id?: string; name: string; adaptor: string; - enabled?: boolean; body: string; delete?: boolean; }; @@ -20,6 +19,7 @@ export type StateEdge = { source_job_id: string | null; source_trigger_id: string | null; target_job_id: string; + enabled?: boolean; }; export type SpecEdge = { @@ -27,6 +27,7 @@ export type SpecEdge = { source_job?: string | null; source_trigger?: string | null; target_job: string | null; + enabled?: boolean; }; export type WorkflowSpec = { diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index 44a2b20b5..bfde7158d 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -55,14 +55,12 @@ export function fullExampleState() { name: 'job a', adaptor: '@openfn/language-common@latest', body: '', - enabled: true, }, 'job-b': { id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37', name: 'job b', adaptor: '@openfn/language-common@latest', body: '', - enabled: true, }, }, triggers: { @@ -78,12 +76,14 @@ export function fullExampleState() { condition: null, source_trigger_id: '71f0cbf1-4d8e-443e-afca-8a479ec281a1', target_job_id: '68e172b8-1cca-4085-aadf-8534761ef7c2', + enabled: true, }, 'job-a->job-b': { id: '7132f768-e8e8-4167-8fc2-8d422244281f', condition: null, source_job_id: '68e172b8-1cca-4085-aadf-8534761ef7c2', target_job_id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37', + enabled: true, }, }, }, @@ -113,50 +113,50 @@ export const lightningProjectPayload = { source_trigger_id: '951fb278-3829-40e6-b86d-c5a6603a0df1', target_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc', condition: 'always', + enabled: true, }, { id: 'a571d495-8f47-4c24-9be4-631eff6e3b8d', target_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', condition: 'on_job_success', source_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc', + enabled: true, }, { id: 'e4a2d2ff-1281-4549-b919-5a6fd369bdc3', target_job_id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', condition: 'on_job_success', source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', + enabled: false, }, { id: 'f7163a97-03c5-4a45-9abf-69f1b771655f', target_job_id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', condition: 'on_job_failure', source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', + enabled: true, }, ], jobs: [ { - enabled: true, id: '8852a349-0936-4141-8c12-d1bfd910e2dc', name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, { - enabled: true, id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, { - enabled: true, id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, { - enabled: true, id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', @@ -183,11 +183,11 @@ export const lightningProjectPayload = { source_trigger_id: '388bbb05-9a88-4493-9ef1-7404274c27b8', target_job_id: '74306d89-2324-4292-9cd4-99450b750050', condition: 'always', + enabled: true, }, ], jobs: [ { - enabled: true, id: '74306d89-2324-4292-9cd4-99450b750050', name: 'New-job', body: "\nget('/myEndpoint', {\n query: {foo: 'bar', a: 1},\n headers: {'content-type': 'application/json'},\n authentication: {username: 'user', password: 'pass'}\n })\n", @@ -226,50 +226,50 @@ export const lightningProjectState = { source_trigger_id: '951fb278-3829-40e6-b86d-c5a6603a0df1', target_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc', condition: 'always', + enabled: true, }, 'FHIR-standard-Data-with-change->Send-to-OpenHIM-to-route-to-SHR': { id: 'a571d495-8f47-4c24-9be4-631eff6e3b8d', target_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', condition: 'on_job_success', source_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc', + enabled: true, }, 'Send-to-OpenHIM-to-route-to-SHR->Notify-CHW-upload-successful': { id: 'e4a2d2ff-1281-4549-b919-5a6fd369bdc3', target_job_id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', condition: 'on_job_success', source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', + enabled: false, }, 'Send-to-OpenHIM-to-route-to-SHR->Notify-CHW-upload-failed': { id: 'f7163a97-03c5-4a45-9abf-69f1b771655f', target_job_id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', condition: 'on_job_failure', source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', + enabled: true, }, }, jobs: { 'FHIR-standard-Data-with-change': { - enabled: true, id: '8852a349-0936-4141-8c12-d1bfd910e2dc', name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, 'Send-to-OpenHIM-to-route-to-SHR': { - enabled: true, id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, 'Notify-CHW-upload-successful': { - enabled: true, id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', }, 'Notify-CHW-upload-failed': { - enabled: true, id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', @@ -296,11 +296,11 @@ export const lightningProjectState = { source_trigger_id: '388bbb05-9a88-4493-9ef1-7404274c27b8', target_job_id: '74306d89-2324-4292-9cd4-99450b750050', condition: 'always', + enabled: true, }, }, jobs: { 'New-job': { - enabled: true, id: '74306d89-2324-4292-9cd4-99450b750050', name: 'New-job', body: "\nget('/myEndpoint', {\n query: {foo: 'bar', a: 1},\n headers: {'content-type': 'application/json'},\n authentication: {username: 'user', password: 'pass'}\n })\n", diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index af14c526a..eb04a1364 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -69,7 +69,6 @@ test('toNextState adding a job', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', - enabled: true, }, }, triggers: { @@ -122,14 +121,12 @@ test('toNextState with empty state', (t) => { adaptor: '@openfn/language-common@latest', name: 'job a', body: '', - enabled: true, }, 'job-b': { id: getItem(result, 'jobs', 'job-b').id, adaptor: '@openfn/language-common@latest', name: 'job b', body: '', - enabled: true, }, }, triggers: { @@ -145,12 +142,14 @@ test('toNextState with empty state', (t) => { condition: null, source_trigger_id: getItem(result, 'triggers', 'trigger-one').id, target_job_id: getItem(result, 'jobs', 'job-a').id, + enabled: true, }, 'job-a->job-b': { id: getItem(result, 'edges', 'job-a->job-b').id, condition: null, source_job_id: getItem(result, 'jobs', 'job-a').id, target_job_id: getItem(result, 'jobs', 'job-b').id, + enabled: true, }, }, }, @@ -175,7 +174,6 @@ test('toNextState with no changes', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', - enabled: true, }, }, triggers: { @@ -285,7 +283,6 @@ test('toNextState with a new job', (t) => { 'job-a': { id: '68e172b8-1cca-4085-aadf-8534761ef7c2', name: 'job a', - enabled: false, body: 'foo()', adaptor: '@openfn/language-adaptor', }, @@ -293,7 +290,6 @@ test('toNextState with a new job', (t) => { id: getItem(result, 'jobs', 'job-b').id, name: 'job b', adaptor: undefined, - enabled: true, body: undefined, }, }, @@ -420,7 +416,6 @@ test('getStateFromProjectPayload with minimal project', (t) => { ], jobs: [ { - enabled: true, id: 'job-1', name: 'My job', body: 'fn(state => state);', @@ -433,6 +428,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { target_job_id: 'job-1', condition: 'on_job_failure', source_trigger_id: 't1', + enabled: true, }, ], }, @@ -456,7 +452,6 @@ test('getStateFromProjectPayload with minimal project', (t) => { }, jobs: { 'My-job': { - enabled: true, id: 'job-1', name: 'My job', body: 'fn(state => state);', @@ -469,6 +464,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { target_job_id: 'job-1', condition: 'on_job_failure', source_trigger_id: 't1', + enabled: true, }, }, }, From 3c2de8587603f976079a35966f8d03dc94610119 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 20 Nov 2023 17:51:41 +0000 Subject: [PATCH 32/45] changeset --- .changeset/warm-brooms-kneel.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/warm-brooms-kneel.md diff --git a/.changeset/warm-brooms-kneel.md b/.changeset/warm-brooms-kneel.md new file mode 100644 index 000000000..f713f368d --- /dev/null +++ b/.changeset/warm-brooms-kneel.md @@ -0,0 +1,5 @@ +--- +'@openfn/deploy': patch +--- + +Move the "enabled" flag from Jobs to Edges, in-line with Lightning changes From 6652626b905418b1be35774637ce3149b71a2fae Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 09:32:07 +0000 Subject: [PATCH 33/45] worker: accept WORKER_REPO_DIR env var --- packages/ws-worker/README.md | 2 +- packages/ws-worker/src/start.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index 7a40c9d06..2089becdc 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -48,7 +48,7 @@ You can start a dev server (which rebuilds on save) by running: pnpm start:watch ``` -This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`. +This will wrap a real runtime engine into the server. It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `WORKER_REPO_DIR` (or a default path in /tmp) ### Disabling auto-fetch diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 3ef446bf5..46b706b69 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -20,6 +20,8 @@ type Args = { capacity?: number; }; +const { WORKER_REPO_DIR, WORKER_SECRET } = process.env; + const args = yargs(hideBin(process.argv)) .command('server', 'Start a ws-worker server') .option('port', { @@ -39,6 +41,7 @@ const args = yargs(hideBin(process.argv)) .option('repo-dir', { alias: 'd', description: 'Path to the runtime repo (where modules will be installed)', + default: WORKER_REPO_DIR, }) .option('secret', { alias: 's', @@ -79,7 +82,6 @@ if (args.lightning === 'mock') { args.secret = 'abdefg'; } } else if (!args.secret) { - const { WORKER_SECRET } = process.env; if (!WORKER_SECRET) { logger.error('WORKER_SECRET is not set'); process.exit(1); From 04afdecb2d296cece7d1d7cb575abb35983357aa Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 09:32:47 +0000 Subject: [PATCH 34/45] engine: remove ENGINE_REPO_DIR (no env var at all for the repo) --- packages/engine-multi/README.md | 2 +- packages/engine-multi/src/api.ts | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md index f0280d455..4705e7864 100644 --- a/packages/engine-multi/README.md +++ b/packages/engine-multi/README.md @@ -65,7 +65,7 @@ The engine has an auto-install feature. This will ensure that all required adapt Blacklisted modules are not installed. -You can pass the local repo dir through the `repoDir` argument in `createEngine`, or by setting the `ENGINE_REPO_DIR` env var. +You can pass a path to local repo dir through the `repoDir` argument in `createEngine`. If no path is provided, it will use a default value (see the logs). ## Resolving Execution Plans diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 35be8fcab..4b25829b1 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -27,6 +27,8 @@ export type RTEOptions = Partial< } >; +const DEFAULT_REPO_DIR = '/tmp/openfn/worker/repo'; + // Create the engine and handle user-facing stuff, like options parsing // and defaulting const createAPI = async function (options: RTEOptions = {}) { @@ -35,13 +37,8 @@ const createAPI = async function (options: RTEOptions = {}) { const logger = options.logger || createLogger('RTE', { level: 'debug' }); if (!repoDir) { - if (process.env.ENGINE_REPO_DIR) { - repoDir = process.env.ENGINE_REPO_DIR; - } else { - repoDir = '/tmp/openfn/repo'; - logger.warn('Using default repodir'); - logger.warn('Set env var ENGINE_REPO_DIR to use a different directory'); - } + repoDir = DEFAULT_REPO_DIR; + logger.warn('Using default repo directory: ', DEFAULT_REPO_DIR); } logger.info('repoDir set to ', repoDir); From 6f78b7ab6c45101cd2af6fbe1039102cf65b14ef Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 09:33:57 +0000 Subject: [PATCH 35/45] changesets --- .changeset/honest-otters-cry.md | 5 +++++ .changeset/two-hounds-listen.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/honest-otters-cry.md create mode 100644 .changeset/two-hounds-listen.md diff --git a/.changeset/honest-otters-cry.md b/.changeset/honest-otters-cry.md new file mode 100644 index 000000000..48b803fc0 --- /dev/null +++ b/.changeset/honest-otters-cry.md @@ -0,0 +1,5 @@ +--- +'@openfn/engine-multi': minor +--- + +Remove ENGINE_REPO_DIR - the repo must be passed directly now diff --git a/.changeset/two-hounds-listen.md b/.changeset/two-hounds-listen.md new file mode 100644 index 000000000..28104d852 --- /dev/null +++ b/.changeset/two-hounds-listen.md @@ -0,0 +1,5 @@ +--- +'@openfn/ws-worker': patch +--- + +Add env var for WORKER_REPO_DIR From 8103200dcb6940a91405fe6c414181d9369bd4aa Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 10:35:28 +0000 Subject: [PATCH 36/45] tests: add failing autoinstall test --- integration-tests/worker/src/factories.ts | 2 +- integration-tests/worker/src/init.ts | 2 +- .../worker/test/autoinstall.test.ts | 62 +++++++++++++++++++ 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 integration-tests/worker/test/autoinstall.test.ts diff --git a/integration-tests/worker/src/factories.ts b/integration-tests/worker/src/factories.ts index 55d359aa8..045852939 100644 --- a/integration-tests/worker/src/factories.ts +++ b/integration-tests/worker/src/factories.ts @@ -1,6 +1,6 @@ import crypto from 'node:crypto'; -export const createAttempt = (triggers, jobs, edges, args) => ({ +export const createAttempt = (triggers, jobs, edges, args = {}) => ({ id: crypto.randomUUID(), triggers, jobs, diff --git a/integration-tests/worker/src/init.ts b/integration-tests/worker/src/init.ts index 0fb084332..2c57c2e61 100644 --- a/integration-tests/worker/src/init.ts +++ b/integration-tests/worker/src/init.ts @@ -18,7 +18,7 @@ export const initWorker = async (lightningPort, engineArgs = {}) => { const workerPort = randomPort(); const engine = await createEngine({ - // logger: createLogger('engine', { level: 'debug' }), + logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), repoDir: path.resolve('./tmp/repo/default'), ...engineArgs, diff --git a/integration-tests/worker/test/autoinstall.test.ts b/integration-tests/worker/test/autoinstall.test.ts new file mode 100644 index 000000000..56b6bba0d --- /dev/null +++ b/integration-tests/worker/test/autoinstall.test.ts @@ -0,0 +1,62 @@ +// stress test for autoinstall +// this could evolve into stress testing, benchmarking or artillery generally? +// Also I may skip this in CI after the issue is fixed + +import test from 'ava'; +import path from 'node:path'; + +import { initLightning, initWorker } from '../src/init'; +import { createAttempt, createJob } from '../src/factories'; + +const generate = (adaptor, version) => { + const specifier = `@openfn/language-${adaptor}@${version}`; + const job = createJob({ + body: `fn(() => ({ data: "${adaptor}" }))`, + adaptor: specifier, + }); + return createAttempt([], [job], []); +}; + +let lightning; +let worker; + +const run = async (attempt) => { + return new Promise(async (done, reject) => { + lightning.on('attempt:complete', (evt) => { + if (attempt.id === evt.attemptId) { + console.log(evt.payload); + done(lightning.getResult(attempt.id)); + } + }); + + lightning.enqueueAttempt(attempt); + }); +}; + +test.before(async () => { + const lightningPort = 4321; + + lightning = initLightning(lightningPort); + + ({ worker } = await initWorker(lightningPort, { + repoDir: path.resolve('tmp/repo/autoinstall'), + })); +}); + +test.after(async () => { + lightning.destroy(); + await worker.destroy(); +}); + +test('autoinstall three things at once', async (t) => { + const a = generate('common', '1.11.1'); + const b = generate('http', '5.0.0'); + const c = generate('googlesheets', '2.2.2'); + + const [ra, rb, rc] = await Promise.all([run(a), run(b), run(c)]); + console.log(ra); + + t.is(ra.data, 'common'); + t.is(rb.data, 'http'); + t.is(rc.data, 'googlesheets'); +}); From 0e5d7365a4e5c65cfef87aec2833db26d03fb05c Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Tue, 21 Nov 2023 10:47:00 +0000 Subject: [PATCH 37/45] start by node by default --- Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8123d3976..8b30ab9df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,4 @@ WORKDIR /app/packages/ws-worker # ------------------------------------------------------------------------------ EXPOSE 2222 -CMD [ "node", "dist/start.js"] - -# TODO: determine how to pass in the -l from `docker run` for running on mac m1 -# CMD [ "pnpm", "start", "-l", "ws://host.docker.internal:4000"] \ No newline at end of file +CMD [ "node", "./dist/start.js"] \ No newline at end of file From 54d00172b76a438cf5ff90a4f1f350ae7438a82f Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Tue, 21 Nov 2023 10:48:35 +0000 Subject: [PATCH 38/45] add changeset --- .changeset/stale-spies-rush.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/stale-spies-rush.md diff --git a/.changeset/stale-spies-rush.md b/.changeset/stale-spies-rush.md new file mode 100644 index 000000000..da1ea9f31 --- /dev/null +++ b/.changeset/stale-spies-rush.md @@ -0,0 +1,5 @@ +--- +'@openfn/ws-worker': patch +--- + +Start ws-worker using node (not pnpm) by default From 378fc6650b09d55ee93fffe2e82f2ec912a8623f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 11:16:30 +0000 Subject: [PATCH 39/45] engine: restructure autoinstall to use a single queue --- packages/engine-multi/src/api/autoinstall.ts | 144 ++++++++++++------ .../engine-multi/test/api/autoinstall.test.ts | 21 ++- 2 files changed, 114 insertions(+), 51 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 881f34de7..3b64221a7 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -1,5 +1,3 @@ -// https://github.com/OpenFn/kit/issues/251 - import { ExecutionPlan, ensureRepo, @@ -9,11 +7,12 @@ import { } from '@openfn/runtime'; import { install as runtimeInstall } from '@openfn/runtime'; -import type { Logger } from '@openfn/logger'; -import type { ExecutionContext } from '../types'; import { AUTOINSTALL_COMPLETE, AUTOINSTALL_ERROR } from '../events'; import { AutoinstallError } from '../errors'; +import type { Logger } from '@openfn/logger'; +import type { ExecutionContext } from '../types'; + // none of these options should be on the plan actually export type AutoinstallOptions = { skipRepoValidation?: boolean; @@ -27,7 +26,74 @@ export type AutoinstallOptions = { const pending: Record> = {}; +let busy = false; + +const queue: Array<{ adaptors: string[]; callback: (err?: any) => void }> = []; + +const enqueue = (adaptors: string[]) => + new Promise((resolve) => { + queue.push({ adaptors, callback: resolve }); + }); + const autoinstall = async (context: ExecutionContext): Promise => { + // TODO not a huge fan of these functions in the closure, but it's ok for now + const processQueue = async () => { + const next = queue.shift(); + if (next) { + busy = true; + const { adaptors, callback } = next; + await doAutoinstall(adaptors, callback); + processQueue(); + } else { + // do nothing + busy = false; + } + }; + + // This will actually do the autoinstall for an attempt + // it will install one or more adaptors + const doAutoinstall = async ( + adaptors: string[], + onComplete: (err?: any) => void + ) => { + // Check whether we still need to do any work + for (const a of adaptors) { + const { name, version } = getNameAndVersion(a); + if (await isInstalledFn(a, repoDir, logger)) { + continue; + } + + const startTime = Date.now(); + try { + await installFn(a, repoDir, logger); + + const duration = Date.now() - startTime; + logger.success(`autoinstalled ${a} in ${duration / 1000}s`); + context.emit(AUTOINSTALL_COMPLETE, { + module: name, + version: version!, + duration, + }); + } catch (e: any) { + delete pending[a]; + + logger.error(`ERROR autoinstalling ${a}: ${e.message}`); + logger.error(e); + const duration = Date.now() - startTime; + context.emit(AUTOINSTALL_ERROR, { + module: name, + version: version!, + duration, + message: e.message || e.toString(), + }); + + // Abort on the first error + return onComplete(new AutoinstallError(a, e)); + } + } + onComplete(); + }; + const { logger, state, options } = context; const { plan } = state; const { repoDir, whitelist } = options; @@ -47,71 +113,49 @@ const autoinstall = async (context: ExecutionContext): Promise => { if (!skipRepoValidation && !didValidateRepo) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate + // TODO do we need to do it on EVERY call? Can we not cache it? await ensureRepo(repoDir, logger); didValidateRepo = true; } const adaptors = Array.from(identifyAdaptors(plan)); - // TODO would rather do all this in parallel but this is fine for now - // TODO set iteration is weirdly difficult? const paths: ModulePaths = {}; + const adaptorsToLoad = []; for (const a of adaptors) { // Ensure that this is not blacklisted - // TODO what if it is? For now we'll log and skip it if (whitelist && !whitelist.find((r) => r.exec(a))) { + // TODO what if it is? For now we'll log and skip it + // TODO actually we should throw a security error in this case logger.warn('WARNING: autoinstall skipping blacklisted module ', a); continue; } - // Return a path name to this module for the linker to use later - // TODO this is all a bit rushed const alias = getAliasedName(a); - const { name, version } = getNameAndVersion(a); + const { name } = getNameAndVersion(a); paths[name] = { path: `${repoDir}/node_modules/${alias}` }; - const needsInstalling = !(await isInstalledFn(a, repoDir, logger)); - if (needsInstalling) { - if (!pending[a]) { - const startTime = Date.now(); - pending[a] = installFn(a, repoDir, logger) - .then(() => { - const duration = Date.now() - startTime; - - logger.success(`autoinstalled ${a} in ${duration / 1000}s`); - context.emit(AUTOINSTALL_COMPLETE, { - module: name, - version: version!, - duration, - }); - delete pending[a]; - }) - .catch((e: any) => { - delete pending[a]; - - logger.error(`ERROR autoinstalling ${a}: ${e.message}`); - logger.error(e); - const duration = Date.now() - startTime; - context.emit(AUTOINSTALL_ERROR, { - module: name, - version: version!, - duration, - message: e.message || e.toString(), - }); - - // wrap and re-throw the error - throw new AutoinstallError(a, e); - }); - } else { - logger.info( - `autoinstall waiting for previous promise for ${a} to resolve...` - ); - } - // Return the pending promise (safe to do this multiple times) - // TODO if this is a chained promise, emit something like "using cache for ${name}" - await pending[a].then(); + if (!(await isInstalledFn(a, repoDir, logger))) { + adaptorsToLoad.push(a); } } + + if (adaptorsToLoad.length) { + // Add this to the queue + const p = enqueue(adaptorsToLoad); + + if (!busy) { + processQueue(); + } + + return p.then((err) => { + if (err) { + throw err; + } + return paths; + }); + } + return paths; }; diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index e3cb73074..d7800a3bd 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -53,6 +53,25 @@ test.afterEach(() => { logger._reset(); }); +// maybe keep this as a simple high level test? +// should work +test.skip('new test', async (t) => { + const autoinstallOpts = { + handleInstall: mockHandleInstall, + handleIsInstalled: async () => false, + }; + const context = createContext(autoinstallOpts); + + const result = await autoinstall(context); + console.log(result); +}); + +// TODO +// error handling +// Queue for multiple installs +// Queue for multiple installs of the same version +// don't autoinstall if it's already there + test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ name: 'repo', @@ -356,7 +375,7 @@ test.serial('autoinstall: emit on error', async (t) => { t.true(!isNaN(evt.duration)); }); -test.serial('autoinstall: throw twice in a ror', async (t) => { +test.serial('autoinstall: throw twice in a row', async (t) => { let callCount = 0; const mockIsInstalled = async () => false; From 948907074ca4b5b784aab567e292dc94acb81719 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 12:20:10 +0000 Subject: [PATCH 40/45] engine: update tests --- packages/engine-multi/src/api/autoinstall.ts | 6 +- .../engine-multi/test/api/autoinstall.test.ts | 109 +++++++++++++++--- 2 files changed, 95 insertions(+), 20 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 3b64221a7..74677c389 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -35,6 +35,9 @@ const enqueue = (adaptors: string[]) => queue.push({ adaptors, callback: resolve }); }); +// Install any modules for an Execution Plan that are not already installed +// This will enforce a queue ensuring only one module is installed at a time +// This fixes https://github.com/OpenFn/kit/issues/503 const autoinstall = async (context: ExecutionContext): Promise => { // TODO not a huge fan of these functions in the closure, but it's ok for now const processQueue = async () => { @@ -50,8 +53,7 @@ const autoinstall = async (context: ExecutionContext): Promise => { } }; - // This will actually do the autoinstall for an attempt - // it will install one or more adaptors + // This will actually do the autoinstall for an attempt (all adaptors) const doAutoinstall = async ( adaptors: string[], onComplete: (err?: any) => void diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index d7800a3bd..323a71633 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -53,25 +53,21 @@ test.afterEach(() => { logger._reset(); }); -// maybe keep this as a simple high level test? -// should work -test.skip('new test', async (t) => { +test('Autoinstall basically works', async (t) => { const autoinstallOpts = { handleInstall: mockHandleInstall, handleIsInstalled: async () => false, }; const context = createContext(autoinstallOpts); - const result = await autoinstall(context); - console.log(result); + const paths = await autoinstall(context); + t.deepEqual(paths, { + '@openfn/language-common': { + path: 'tmp/repo/node_modules/@openfn/language-common_1.0.0', + }, + }); }); -// TODO -// error handling -// Queue for multiple installs -// Queue for multiple installs of the same version -// don't autoinstall if it's already there - test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ name: 'repo', @@ -147,13 +143,22 @@ test.serial('autoinstall: should call both mock functions', async (t) => { t.true(didCallInstall); }); +// TODO +// error handling +// Queue for multiple installs +// Queue for multiple installs of the same version +// don't autoinstall if it's already there + test.serial( 'autoinstall: only call install once if there are two concurrent install requests', async (t) => { let callCount = 0; - const mockInstall = () => + const installed = {}; + + const mockInstall = (name) => new Promise((resolve) => { + installed[name] = true; callCount++; setTimeout(() => resolve(), 20); }); @@ -161,7 +166,7 @@ test.serial( const options = { skipRepoValidation: true, handleInstall: mockInstall, - handleIsInstalled: async () => false, + handleIsInstalled: async (name) => name in installed, }; const context = createContext(options); @@ -172,6 +177,78 @@ test.serial( } ); +test.serial('autoinstall: install in sequence', async (t) => { + const installed = {}; + + const states = {}; + + const mockInstall = (name) => + new Promise((resolve) => { + // Each time install is called, + // record the time the call was made + // and the install state + states[name] = { + time: new Date().getTime(), + installed: Object.keys(installed).map((s) => s.split('common@')[1]), + }; + installed[name] = true; + setTimeout(() => resolve(), 50); + }); + + const options = { + skipRepoValidation: true, + handleInstall: mockInstall, + handleIsInstalled: false, + }; + + const c1 = createContext(options, [{ adaptor: '@openfn/language-common@1' }]); + const c2 = createContext(options, [{ adaptor: '@openfn/language-common@2' }]); + const c3 = createContext(options, [{ adaptor: '@openfn/language-common@3' }]); + + await Promise.all([autoinstall(c1), autoinstall(c2), autoinstall(c3)]); + + const s1 = states['@openfn/language-common@1']; + const s2 = states['@openfn/language-common@2']; + const s3 = states['@openfn/language-common@3']; + + // Check that modules are installed in sequence + t.deepEqual(s1.installed, []); + t.deepEqual(s2.installed, ['1']); + t.deepEqual(s3.installed, ['1', '2']); + + // And check for a time gap between installs + t.true(s3.time - s2.time > 40); + t.true(s2.time - s1.time > 40); +}); + +test('autoinstall: handle two seperate, non-overlapping installs', async (t) => { + const options = { + handleInstall: mockHandleInstall, + handleIsInstalled: async () => false, + }; + + const c1 = createContext(options, [ + { adaptor: '@openfn/language-dhis2@1.0.0' }, + ]); + const c2 = createContext(options, [ + { adaptor: '@openfn/language-http@1.0.0' }, + ]); + + const p1 = await autoinstall(c1); + t.deepEqual(p1, { + '@openfn/language-dhis2': { + path: 'tmp/repo/node_modules/@openfn/language-dhis2_1.0.0', + }, + }); + + const p2 = await autoinstall(c2); + t.deepEqual(p2, { + '@openfn/language-http': { + path: 'tmp/repo/node_modules/@openfn/language-http_1.0.0', + }, + }); +}); + test.serial( 'autoinstall: do not try to install blacklisted modules', async (t) => { @@ -330,15 +407,11 @@ test.serial('autoinstall: throw on error twice if pending', async (t) => { autoinstall(context).catch(assertCatches); - // The two catches won't neccessarily return in order - // (shrug asynchronous code?) - // So this catch-all callback will resolve the test when both - // promises have resolved function assertCatches(e) { t.is(e.name, 'AutoinstallError'); errCount += 1; if (errCount === 2) { - t.is(callCount, 1); + t.is(callCount, 2); t.pass('threw twice!'); done(); } From 7cc8ec07ffb5ef8ee2f27f7f6f7e8b7403700719 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 12:22:01 +0000 Subject: [PATCH 41/45] tests: tidy --- integration-tests/worker/src/init.ts | 2 +- integration-tests/worker/test/autoinstall.test.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-tests/worker/src/init.ts b/integration-tests/worker/src/init.ts index 2c57c2e61..0fb084332 100644 --- a/integration-tests/worker/src/init.ts +++ b/integration-tests/worker/src/init.ts @@ -18,7 +18,7 @@ export const initWorker = async (lightningPort, engineArgs = {}) => { const workerPort = randomPort(); const engine = await createEngine({ - logger: createLogger('engine', { level: 'debug' }), + // logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), repoDir: path.resolve('./tmp/repo/default'), ...engineArgs, diff --git a/integration-tests/worker/test/autoinstall.test.ts b/integration-tests/worker/test/autoinstall.test.ts index 56b6bba0d..80bdaddf5 100644 --- a/integration-tests/worker/test/autoinstall.test.ts +++ b/integration-tests/worker/test/autoinstall.test.ts @@ -24,7 +24,6 @@ const run = async (attempt) => { return new Promise(async (done, reject) => { lightning.on('attempt:complete', (evt) => { if (attempt.id === evt.attemptId) { - console.log(evt.payload); done(lightning.getResult(attempt.id)); } }); @@ -54,7 +53,6 @@ test('autoinstall three things at once', async (t) => { const c = generate('googlesheets', '2.2.2'); const [ra, rb, rc] = await Promise.all([run(a), run(b), run(c)]); - console.log(ra); t.is(ra.data, 'common'); t.is(rb.data, 'http'); From 4a17048a3ce379021d1fd615fa3fec518fe78bbe Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 12:31:34 +0000 Subject: [PATCH 42/45] changeset --- .changeset/moody-bulldogs-camp.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/moody-bulldogs-camp.md diff --git a/.changeset/moody-bulldogs-camp.md b/.changeset/moody-bulldogs-camp.md new file mode 100644 index 000000000..2c68c2f24 --- /dev/null +++ b/.changeset/moody-bulldogs-camp.md @@ -0,0 +1,5 @@ +--- +'@openfn/engine-multi': patch +--- + +Queue autoinstall requests to ensure we only install one thing at a time From 898a70d83fbd1aa45164ad5cbe9ebb5fed159ced Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 12:56:50 +0000 Subject: [PATCH 43/45] deploy: support enabled flag on triggers --- packages/deploy/src/stateTransform.ts | 7 ++++--- packages/deploy/src/types.ts | 1 + packages/deploy/test/fixtures.ts | 2 ++ packages/deploy/test/stateTransform.test.ts | 7 +++++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index c7a0424d1..35e1ebb9b 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -71,11 +71,11 @@ function pickValue( key: string, defaultValue: any ): any { - if (typeof first[key] !== 'undefined') { + if (key in first) { return first[key]; } - if (second && typeof second[key] !== 'undefined') { + if (key in second) { return second[key]; } @@ -94,7 +94,7 @@ function mergeTriggers( triggerKey, { id: crypto.randomUUID(), - ...pickKeys(specTrigger, ['type']), + ...pickKeys(specTrigger, ['type', 'enabled']), ...(specTrigger.type === 'cron' ? { cron_expression: specTrigger.cron_expression } : {}), @@ -113,6 +113,7 @@ function mergeTriggers( id: stateTrigger!.id, ...{ type: pickValue(specTrigger!, stateTrigger!, 'type', 'webhook'), + enabled: pickValue(specTrigger!, stateTrigger!, 'enabled', true), ...(specTrigger!.type === 'cron' ? { cron_expression: specTrigger!.cron_expression } : {}), diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 992ff7774..5df06762d 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -11,6 +11,7 @@ export type Trigger = { type?: string; cron_expression?: string; delete?: boolean; + enabled?: boolean; }; export type StateEdge = { diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index bfde7158d..c2e929041 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -23,6 +23,7 @@ export function fullExampleSpec() { 'trigger-one': { type: 'cron', cron_expression: '0 0 1 1 *', + enabled: true, }, }, edges: { @@ -68,6 +69,7 @@ export function fullExampleState() { id: '71f0cbf1-4d8e-443e-afca-8a479ec281a1', type: 'cron', cron_expression: '0 0 1 1 *', + enabled: true, }, }, edges: { diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index eb04a1364..e88249eba 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -31,6 +31,7 @@ test('toNextState adding a job', (t) => { 'trigger-one': { type: 'cron', cron_expression: '0 0 1 1 *', + enabled: false, }, }, edges: {}, @@ -46,6 +47,7 @@ test('toNextState adding a job', (t) => { triggers: { 'trigger-one': { id: '57912d4a-13e5-4857-8e1b-473be3816fd8', + enabled: true, }, }, edges: {}, @@ -76,6 +78,7 @@ test('toNextState adding a job', (t) => { id: '57912d4a-13e5-4857-8e1b-473be3816fd8', type: 'cron', cron_expression: '0 0 1 1 *', + enabled: false, }, }, edges: {}, @@ -134,6 +137,7 @@ test('toNextState with empty state', (t) => { id: getItem(result, 'triggers', 'trigger-one').id, type: 'cron', cron_expression: '0 0 1 1 *', + enabled: true, }, }, edges: { @@ -181,6 +185,7 @@ test('toNextState with no changes', (t) => { id: '71f0cbf1-4d8e-443e-afca-8a479ec281a1', type: 'cron', cron_expression: '0 0 1 1 *', + enabled: true, }, }, edges: {}, @@ -205,6 +210,7 @@ test('toNextState with no changes', (t) => { 'trigger-one': { type: 'cron', cron_expression: '0 0 1 1 *', + enabled: true, }, }, edges: {}, @@ -297,6 +303,7 @@ test('toNextState with a new job', (t) => { 'trigger-one': { id: '71f0cbf1-4d8e-443e-afca-8a479ec281a1', type: 'webhook', + enabled: true, }, }, edges: {}, From 0c652303538443d0733c2fa69a0347d35523f24c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 12:58:01 +0000 Subject: [PATCH 44/45] update changeset --- .changeset/warm-brooms-kneel.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/warm-brooms-kneel.md b/.changeset/warm-brooms-kneel.md index f713f368d..126923a38 100644 --- a/.changeset/warm-brooms-kneel.md +++ b/.changeset/warm-brooms-kneel.md @@ -2,4 +2,4 @@ '@openfn/deploy': patch --- -Move the "enabled" flag from Jobs to Edges, in-line with Lightning changes +Remove the `enabled` flag from Jobs, and add to Triggers and Edges From 35c6c426a6c0cb0fc86c3df6b11519259ff39de6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 21 Nov 2023 13:20:25 +0000 Subject: [PATCH 45/45] release worker@0.2.9 cli@0.4.9 --- .changeset/honest-otters-cry.md | 5 - .changeset/lovely-dodos-guess.md | 5 - .changeset/moody-bulldogs-camp.md | 5 - .changeset/stale-spies-rush.md | 5 - .changeset/two-hounds-listen.md | 5 - .changeset/warm-brooms-kneel.md | 5 - integration-tests/worker/CHANGELOG.md | 12 + integration-tests/worker/package.json | 2 +- packages/cli/CHANGELOG.md | 8 + packages/cli/package.json | 2 +- packages/deploy/CHANGELOG.md | 6 + packages/deploy/package.json | 2 +- packages/engine-multi/CHANGELOG.md | 12 + packages/engine-multi/package.json | 2 +- packages/lightning-mock/CHANGELOG.md | 8 + packages/lightning-mock/package.json | 2 +- packages/runtime/CHANGELOG.md | 6 + packages/runtime/package.json | 2 +- packages/ws-worker/CHANGELOG.md | 10 + packages/ws-worker/package.json | 2 +- pnpm-lock.yaml | 678 +++++++++++++++++++++++++- 21 files changed, 730 insertions(+), 54 deletions(-) delete mode 100644 .changeset/honest-otters-cry.md delete mode 100644 .changeset/lovely-dodos-guess.md delete mode 100644 .changeset/moody-bulldogs-camp.md delete mode 100644 .changeset/stale-spies-rush.md delete mode 100644 .changeset/two-hounds-listen.md delete mode 100644 .changeset/warm-brooms-kneel.md diff --git a/.changeset/honest-otters-cry.md b/.changeset/honest-otters-cry.md deleted file mode 100644 index 48b803fc0..000000000 --- a/.changeset/honest-otters-cry.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/engine-multi': minor ---- - -Remove ENGINE_REPO_DIR - the repo must be passed directly now diff --git a/.changeset/lovely-dodos-guess.md b/.changeset/lovely-dodos-guess.md deleted file mode 100644 index cf75f4a47..000000000 --- a/.changeset/lovely-dodos-guess.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/runtime': minor ---- - -Allow globals to be passed into the execution environment diff --git a/.changeset/moody-bulldogs-camp.md b/.changeset/moody-bulldogs-camp.md deleted file mode 100644 index 2c68c2f24..000000000 --- a/.changeset/moody-bulldogs-camp.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/engine-multi': patch ---- - -Queue autoinstall requests to ensure we only install one thing at a time diff --git a/.changeset/stale-spies-rush.md b/.changeset/stale-spies-rush.md deleted file mode 100644 index da1ea9f31..000000000 --- a/.changeset/stale-spies-rush.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/ws-worker': patch ---- - -Start ws-worker using node (not pnpm) by default diff --git a/.changeset/two-hounds-listen.md b/.changeset/two-hounds-listen.md deleted file mode 100644 index 28104d852..000000000 --- a/.changeset/two-hounds-listen.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/ws-worker': patch ---- - -Add env var for WORKER_REPO_DIR diff --git a/.changeset/warm-brooms-kneel.md b/.changeset/warm-brooms-kneel.md deleted file mode 100644 index 126923a38..000000000 --- a/.changeset/warm-brooms-kneel.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/deploy': patch ---- - -Remove the `enabled` flag from Jobs, and add to Triggers and Edges diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 3109aaf64..fe8c16930 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,17 @@ # @openfn/integration-tests-worker +## 1.0.20 + +### Patch Changes + +- Updated dependencies [6f78b7a] +- Updated dependencies [4a17048] +- Updated dependencies [54d0017] +- Updated dependencies [6f78b7a] + - @openfn/engine-multi@0.2.0 + - @openfn/ws-worker@0.2.9 + - @openfn/lightning-mock@1.1.2 + ## 1.0.19 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 0d0f4b60e..79d2be022 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.19", + "version": "1.0.20", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 232c98df7..42f53349d 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,13 @@ # @openfn/cli +## 0.4.9 + +### Patch Changes + +- Updated dependencies [3c2de85] + - @openfn/runtime@0.2.0 + - @openfn/deploy@0.2.10 + ## 0.4.8 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index a70c5763a..437f7ec5c 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "0.4.8", + "version": "0.4.9", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index 3e310a1e6..857e371e6 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/deploy +## 0.2.10 + +### Patch Changes + +- 3c2de85: Remove the `enabled` flag from Jobs, and add to Triggers and Edges + ## 0.2.9 ### Patch Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index 21d331281..29aac14e2 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.2.9", + "version": "0.2.10", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 64134dad1..2d3947592 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,17 @@ # engine-multi +## 0.2.0 + +### Minor Changes + +- 6f78b7a: Remove ENGINE_REPO_DIR - the repo must be passed directly now + +### Patch Changes + +- 4a17048: Queue autoinstall requests to ensure we only install one thing at a time +- Updated dependencies [40ffc22] + - @openfn/runtime@0.2.0 + ## 0.1.11 ### Patch Changes diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index c5344df41..2337f2482 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "0.1.11", + "version": "0.2.0", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index 82e4aa585..7bc10a6b4 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,13 @@ # @openfn/lightning-mock +## 1.1.2 + +### Patch Changes + +- Updated dependencies [4a17048] + - @openfn/engine-multi@0.2.0 + - @openfn/runtime@0.2.0 + ## 1.1.1 ### Patch Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 8b0bb45d1..aee2eb260 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.1.1", + "version": "1.1.2", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 451706128..8d275d366 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/runtime +## 0.2.0 + +### Minor Changes + +- 40ffc22: Allow globals to be passed into the execution environment + ## 0.1.4 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index aba0e2231..057281117 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "0.1.4", + "version": "0.2.0", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index 544b78389..0786c3e44 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,15 @@ # ws-worker +## 0.2.9 + +### Patch Changes + +- 54d0017: Start ws-worker using node (not pnpm) by default +- 6f78b7a: Add env var for WORKER_REPO_DIR +- Updated dependencies [4a17048] + - @openfn/engine-multi@0.2.0 + - @openfn/runtime@0.2.0 + ## 0.2.8 ### Patch Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 8c0461963..b58173c98 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "0.2.8", + "version": "0.2.9", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7aeceb6ab..c125e5dcb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,6 +146,18 @@ importers: specifier: ^5.1.6 version: 5.1.6 + integration-tests/worker/tmp/repo/autoinstall: + dependencies: + '@openfn/language-common_1.11.1': + specifier: npm:@openfn/language-common@^1.11.1 + version: /@openfn/language-common@1.11.1 + '@openfn/language-googlesheets_2.2.2': + specifier: npm:@openfn/language-googlesheets@^2.2.2 + version: /@openfn/language-googlesheets@2.2.2 + '@openfn/language-http_5.0.0': + specifier: npm:@openfn/language-http@^5.0.0 + version: /@openfn/language-http@5.0.0 + packages/cli: dependencies: '@inquirer/prompts': @@ -1330,6 +1342,11 @@ packages: heap: 0.2.7 dev: false + /@fastify/busboy@2.1.0: + resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==} + engines: {node: '>=14'} + dev: false + /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1586,6 +1603,34 @@ packages: semver: 7.5.4 dev: true + /@openfn/language-common@1.10.1: + resolution: {integrity: sha512-LTH9arUPPzbmmswVrLp9pFxrjYeo9rJB0UMA0yZ5tlU2CKnr7Pj35YkxlpmdcHMBvMo+tCrDrL89MjwQhwyzDA==} + dependencies: + axios: 1.1.3 + csv-parse: 5.5.2 + csvtojson: 2.0.10 + date-fns: 2.30.0 + jsonpath-plus: 4.0.0 + lodash: 4.17.21 + transitivePeerDependencies: + - debug + dev: false + + /@openfn/language-common@1.11.1: + resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==} + dependencies: + ajv: 8.12.0 + axios: 1.1.3 + csv-parse: 5.5.2 + csvtojson: 2.0.10 + date-fns: 2.30.0 + jsonpath-plus: 4.0.0 + lodash: 4.17.21 + undici: 5.27.2 + transitivePeerDependencies: + - debug + dev: false + /@openfn/language-common@1.7.5: resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==} dependencies: @@ -1601,6 +1646,33 @@ packages: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] + /@openfn/language-googlesheets@2.2.2: + resolution: {integrity: sha512-Ez7M1w/gtJCZjnpbebHzwGJTmL7JmBkKG9It1Mxu8lTjVgrz1Tr5yJLWRE1HgAsiQjDjF6l2WidnXy6lHbkdbw==} + dependencies: + '@openfn/language-common': 1.11.1 + googleapis: 100.0.0 + transitivePeerDependencies: + - debug + - encoding + - supports-color + dev: false + + /@openfn/language-http@5.0.0: + resolution: {integrity: sha512-UUsazztKd6h0z61OR9hyurICRhRiD9yQaJV3mNPJkDtnWZubpYDZg+JWAatgPJHe2K0u+Mb18coAGhVLHb5O3A==} + dependencies: + '@openfn/language-common': 1.10.1 + cheerio: 1.0.0-rc.12 + cheerio-tableparser: 1.0.1 + csv-parse: 4.16.3 + fast-safe-stringify: 2.1.1 + form-data: 3.0.1 + lodash: 4.17.21 + request: 2.88.2 + tough-cookie: 4.1.3 + transitivePeerDependencies: + - debug + dev: false + /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1969,7 +2041,6 @@ packages: engines: {node: '>=6.5'} dependencies: event-target-shim: 5.0.1 - dev: true /accepts@1.3.8: resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} @@ -2019,7 +2090,6 @@ packages: debug: 4.3.4 transitivePeerDependencies: - supports-color - dev: true /agentkeepalive@4.3.0: resolution: {integrity: sha512-7Epl1Blf4Sy37j4v9f9FjICCh4+KAQOyXgHEwlyBiAQLbhKdq/i2QQU3amQalS/wPhdPzDXPL5DMR5bkn+YeWg==} @@ -2047,6 +2117,24 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 + /ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + dev: false + + /ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + dev: false + /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -2178,10 +2266,26 @@ packages: engines: {node: '>=0.10.0'} dev: true + /arrify@2.0.1: + resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==} + engines: {node: '>=8'} + dev: false + /arrify@3.0.0: resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} engines: {node: '>=12'} + /asn1@0.2.6: + resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /assert-plus@1.0.0: + resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} + engines: {node: '>=0.8'} + dev: false + /assign-symbols@1.0.0: resolution: {integrity: sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==} engines: {node: '>=0.10.0'} @@ -2207,7 +2311,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -2338,6 +2441,14 @@ packages: fast-glob: 3.3.1 dev: true + /aws-sign2@0.7.0: + resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} + dev: false + + /aws4@1.12.0: + resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} + dev: false + /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: @@ -2355,7 +2466,6 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - dev: true /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -2366,7 +2476,6 @@ packages: /base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} - dev: true /base@0.11.2: resolution: {integrity: sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==} @@ -2392,6 +2501,12 @@ packages: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} dev: true + /bcrypt-pbkdf@1.0.2: + resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} + dependencies: + tweetnacl: 0.14.5 + dev: false + /bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: true @@ -2403,6 +2518,10 @@ packages: is-windows: 1.0.2 dev: true + /bignumber.js@9.1.2: + resolution: {integrity: sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==} + dev: false + /binary-extensions@1.13.1: resolution: {integrity: sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==} engines: {node: '>=0.10.0'} @@ -2436,9 +2555,17 @@ packages: readable-stream: 4.2.0 dev: true + /bluebird@3.7.2: + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} + dev: false + /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2487,6 +2614,10 @@ packages: pako: 0.2.9 dev: true + /buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + dev: false + /buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} dev: true @@ -2623,6 +2754,10 @@ packages: engines: {node: '>=6'} dev: true + /caseless@0.12.0: + resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + dev: false + /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -2656,6 +2791,34 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio-tableparser@1.0.1: + resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -2832,7 +2995,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -2907,6 +3069,10 @@ packages: resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} dev: false + /core-util-is@1.0.2: + resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} + dev: false + /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -2957,6 +3123,21 @@ packages: which: 2.0.2 dev: true + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -2973,7 +3154,10 @@ packages: /csv-parse@4.16.3: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} - dev: true + + /csv-parse@5.5.2: + resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} + dev: false /csv-stringify@5.6.5: resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==} @@ -2989,12 +3173,29 @@ packages: stream-transform: 2.1.3 dev: true + /csvtojson@2.0.10: + resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} + engines: {node: '>=4.0.0'} + hasBin: true + dependencies: + bluebird: 3.7.2 + lodash: 4.17.21 + strip-bom: 2.0.0 + dev: false + /currently-unhandled@0.4.1: resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==} engines: {node: '>=0.10.0'} dependencies: array-find-index: 1.0.2 + /dashdash@1.14.1: + resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} + engines: {node: '>=0.10'} + dependencies: + assert-plus: 1.0.0 + dev: false + /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -3129,7 +3330,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3180,6 +3380,33 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3203,6 +3430,19 @@ packages: /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + /ecc-jsbn@0.1.2: + resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} + dependencies: + jsbn: 0.1.1 + safer-buffer: 2.1.2 + dev: false + + /ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + dependencies: + safe-buffer: 5.2.1 + dev: false + /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3241,6 +3481,11 @@ packages: ansi-colors: 4.1.3 dev: true + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -3867,7 +4112,6 @@ packages: /event-target-shim@5.0.1: resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} engines: {node: '>=6'} - dev: true /eventemitter3@3.1.2: resolution: {integrity: sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==} @@ -3927,6 +4171,10 @@ packages: is-extendable: 1.0.1 dev: true + /extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + dev: false + /extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} dev: true @@ -3955,6 +4203,15 @@ packages: - supports-color dev: true + /extsprintf@1.3.0: + resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} + engines: {'0': node >=0.6.0} + dev: false + + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: false + /fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} @@ -3987,6 +4244,10 @@ packages: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} dev: false + /fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + dev: false + /fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} dev: true @@ -3995,6 +4256,10 @@ packages: resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} dev: false + /fast-text-encoding@1.0.6: + resolution: {integrity: sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==} + dev: false + /fastq@1.13.0: resolution: {integrity: sha512-YpkpUnK8od0o1hmeSc7UUs/eB/vIPWJYjKck2QKIzAf71Vm1AAQ3EbuZB3g2JIy+pg+ERD0vqI79KyZiB2e2Nw==} dependencies: @@ -4111,7 +4376,6 @@ packages: peerDependenciesMeta: debug: optional: true - dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4126,6 +4390,19 @@ packages: signal-exit: 4.0.2 dev: true + /forever-agent@0.6.1: + resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} + dev: false + + /form-data@2.3.3: + resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} + engines: {node: '>= 0.12'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data@2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -4135,6 +4412,15 @@ packages: mime-types: 2.1.35 dev: true + /form-data@3.0.1: + resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4142,7 +4428,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -4230,6 +4515,31 @@ packages: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} dev: true + /gaxios@4.3.3: + resolution: {integrity: sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA==} + engines: {node: '>=10'} + dependencies: + abort-controller: 3.0.0 + extend: 3.0.2 + https-proxy-agent: 5.0.1 + is-stream: 2.0.1 + node-fetch: 2.6.7 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + + /gcp-metadata@4.3.1: + resolution: {integrity: sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A==} + engines: {node: '>=10'} + dependencies: + gaxios: 4.3.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + /get-caller-file@2.0.5: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} @@ -4259,6 +4569,12 @@ packages: engines: {node: '>=0.10.0'} dev: true + /getpass@0.1.7: + resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} + dependencies: + assert-plus: 1.0.0 + dev: false + /glob-parent@3.1.0: resolution: {integrity: sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==} dependencies: @@ -4345,6 +4661,58 @@ packages: merge2: 1.4.1 slash: 4.0.0 + /google-auth-library@7.14.1: + resolution: {integrity: sha512-5Rk7iLNDFhFeBYc3s8l1CqzbEBcdhwR193RlD4vSNFajIcINKI8W8P0JLmBpwymHqqWbX34pJDQu39cSy/6RsA==} + engines: {node: '>=10'} + dependencies: + arrify: 2.0.1 + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + fast-text-encoding: 1.0.6 + gaxios: 4.3.3 + gcp-metadata: 4.3.1 + gtoken: 5.3.2 + jws: 4.0.0 + lru-cache: 6.0.0 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + + /google-p12-pem@3.1.4: + resolution: {integrity: sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg==} + engines: {node: '>=10'} + hasBin: true + dependencies: + node-forge: 1.3.1 + dev: false + + /googleapis-common@5.1.0: + resolution: {integrity: sha512-RXrif+Gzhq1QAzfjxulbGvAY3FPj8zq/CYcvgjzDbaBNCD6bUl+86I7mUs4DKWHGruuK26ijjR/eDpWIDgNROA==} + engines: {node: '>=10.10.0'} + dependencies: + extend: 3.0.2 + gaxios: 4.3.3 + google-auth-library: 7.14.1 + qs: 6.11.2 + url-template: 2.0.8 + uuid: 8.3.2 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + + /googleapis@100.0.0: + resolution: {integrity: sha512-RToFQGY54B756IDbjdyjb1vWFmn03bYpXHB2lIf0eq2UBYsIbYOLZ0kqSomfJnpclEukwEmMF7Jn6Wsev871ew==} + engines: {node: '>=10'} + dependencies: + google-auth-library: 7.14.1 + googleapis-common: 5.1.0 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + /graceful-fs@4.2.10: resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} dev: true @@ -4353,6 +4721,18 @@ packages: resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==} dev: true + /gtoken@5.3.2: + resolution: {integrity: sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ==} + engines: {node: '>=10'} + dependencies: + gaxios: 4.3.3 + google-p12-pem: 3.1.4 + jws: 4.0.0 + transitivePeerDependencies: + - encoding + - supports-color + dev: false + /gunzip-maybe@1.4.2: resolution: {integrity: sha512-4haO1M4mLO91PW57BMsDFf75UmwoRX0GkdD+Faw+Lr+r/OZrOCS0pIBwOL1xCKQqnQzbNFGgK2V2CpBUPeFNTw==} hasBin: true @@ -4365,6 +4745,20 @@ packages: through2: 2.0.5 dev: true + /har-schema@2.0.0: + resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} + engines: {node: '>=4'} + dev: false + + /har-validator@5.1.5: + resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} + engines: {node: '>=6'} + deprecated: this library is no longer supported + dependencies: + ajv: 6.12.6 + har-schema: 2.0.0 + dev: false + /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -4450,6 +4844,15 @@ packages: lru-cache: 7.18.3 dev: true + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4518,6 +4921,15 @@ packages: - supports-color dev: true + /http-signature@1.2.0: + resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} + engines: {node: '>=0.8', npm: '>=1.3.7'} + dependencies: + assert-plus: 1.0.0 + jsprim: 1.4.2 + sshpk: 1.18.0 + dev: false + /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -4526,7 +4938,6 @@ packages: debug: 4.3.4 transitivePeerDependencies: - supports-color - dev: true /human-id@1.0.2: resolution: {integrity: sha512-UNopramDEhHJD+VR+ehk8rOslwSfByxPIZyJRfV739NDhN5LF1fa1MqnzKm2lGTQRjNrjK19Q5fhkgIfjlVUKw==} @@ -4893,7 +5304,6 @@ packages: /is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} - dev: true /is-string@1.0.7: resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} @@ -4916,6 +5326,10 @@ packages: has-symbols: 1.0.3 dev: true + /is-typedarray@1.0.0: + resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} + dev: false + /is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -4925,6 +5339,10 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} + /is-utf8@0.2.1: + resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} + dev: false + /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -4965,6 +5383,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /isstream@0.1.2: + resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} + dev: false + /jackspeak@2.2.2: resolution: {integrity: sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==} engines: {node: '>=14'} @@ -5005,6 +5427,16 @@ packages: argparse: 2.0.1 dev: true + /jsbn@0.1.1: + resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} + dev: false + + /json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + dependencies: + bignumber.js: 9.1.2 + dev: false + /json-diff@1.0.6: resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} hasBin: true @@ -5018,6 +5450,22 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true + /json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + dev: false + + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + dev: false + + /json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + dev: false + + /json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + dev: false + /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5032,7 +5480,6 @@ packages: /jsonpath-plus@4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} - dev: true /jsonpath@1.1.1: resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} @@ -5042,6 +5489,31 @@ packages: underscore: 1.12.1 dev: true + /jsprim@1.4.2: + resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} + engines: {node: '>=0.6.0'} + dependencies: + assert-plus: 1.0.0 + extsprintf: 1.3.0 + json-schema: 0.4.0 + verror: 1.10.0 + dev: false + + /jwa@2.0.0: + resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==} + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + dev: false + + /jws@4.0.0: + resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==} + dependencies: + jwa: 2.0.0 + safe-buffer: 5.2.1 + dev: false + /keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} @@ -5684,6 +6156,11 @@ packages: whatwg-url: 5.0.0 dev: false + /node-forge@1.3.1: + resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} + engines: {node: '>= 6.13.0'} + dev: false + /nodemon@3.0.1: resolution: {integrity: sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==} engines: {node: '>=10'} @@ -5764,6 +6241,16 @@ packages: path-key: 3.1.1 dev: true + /nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + dependencies: + boolbase: 1.0.0 + dev: false + + /oauth-sign@0.9.0: + resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6033,6 +6520,19 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6115,6 +6615,10 @@ packages: through2: 2.0.5 dev: true + /performance-now@2.1.0: + resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} + dev: false + /phoenix@1.7.7: resolution: {integrity: sha512-moAN6e4Z16x/x1nswUpnTR2v5gm7HsI7eluZ2YnYUUsBNzi3cY/5frmiJfXIEi877IQAafzTfp8hd6vEUMme+w==} dev: false @@ -6339,7 +6843,6 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -6350,6 +6853,10 @@ packages: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} dev: true + /psl@1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + dev: false + /pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -6372,7 +6879,6 @@ packages: /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} - dev: true /qs@6.11.2: resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} @@ -6381,6 +6887,11 @@ packages: side-channel: 1.0.4 dev: false + /qs@6.5.3: + resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} + engines: {node: '>=0.6'} + dev: false + /query-string@8.1.0: resolution: {integrity: sha512-BFQeWxJOZxZGix7y+SByG3F36dA0AbTy9o6pSmKFcFz7DAj0re9Frkty3saBn3nHo3D0oZJ/+rx3r8H8r8Jbpw==} engines: {node: '>=14.16'} @@ -6390,6 +6901,10 @@ packages: split-on-first: 3.0.0 dev: true + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} + dev: false + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6574,14 +7089,50 @@ packages: engines: {node: '>=0.10'} dev: true + /request@2.88.2: + resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} + engines: {node: '>= 6'} + deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 + dependencies: + aws-sign2: 0.7.0 + aws4: 1.12.0 + caseless: 0.12.0 + combined-stream: 1.0.8 + extend: 3.0.2 + forever-agent: 0.6.1 + form-data: 2.3.3 + har-validator: 5.1.5 + http-signature: 1.2.0 + is-typedarray: 1.0.0 + isstream: 0.1.2 + json-stringify-safe: 5.0.1 + mime-types: 2.1.35 + oauth-sign: 0.9.0 + performance-now: 2.1.0 + qs: 6.5.3 + safe-buffer: 5.2.1 + tough-cookie: 2.5.0 + tunnel-agent: 0.6.0 + uuid: 3.4.0 + dev: false + /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -6993,6 +7544,22 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + /sshpk@1.18.0: + resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} + engines: {node: '>=0.10.0'} + hasBin: true + dependencies: + asn1: 0.2.6 + assert-plus: 1.0.0 + bcrypt-pbkdf: 1.0.2 + dashdash: 1.14.1 + ecc-jsbn: 0.1.2 + getpass: 0.1.7 + jsbn: 0.1.1 + safer-buffer: 2.1.2 + tweetnacl: 0.14.5 + dev: false + /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7101,6 +7668,13 @@ packages: dependencies: ansi-regex: 6.0.1 + /strip-bom@2.0.0: + resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} + engines: {node: '>=0.10.0'} + dependencies: + is-utf8: 0.2.1 + dev: false + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -7317,6 +7891,24 @@ packages: nopt: 1.0.10 dev: true + /tough-cookie@2.5.0: + resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} + engines: {node: '>=0.8'} + dependencies: + psl: 1.9.0 + punycode: 2.3.0 + dev: false + + /tough-cookie@4.1.3: + resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} + engines: {node: '>=6'} + dependencies: + psl: 1.9.0 + punycode: 2.3.0 + universalify: 0.2.0 + url-parse: 1.5.10 + dev: false + /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false @@ -7576,6 +8168,16 @@ packages: yargs: 17.7.2 dev: true + /tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /tweetnacl@0.14.5: + resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} + dev: false + /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -7654,6 +8256,13 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true + /undici@5.27.2: + resolution: {integrity: sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==} + engines: {node: '>=14.0'} + dependencies: + '@fastify/busboy': 2.1.0 + dev: false + /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'} @@ -7683,6 +8292,11 @@ packages: engines: {node: '>= 4.0.0'} dev: true + /universalify@0.2.0: + resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} + engines: {node: '>= 4.0.0'} + dev: false + /unix-crypt-td-js@1.1.4: resolution: {integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==} dev: true @@ -7704,6 +8318,12 @@ packages: engines: {node: '>=4'} dev: true + /uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + dependencies: + punycode: 2.3.0 + dev: false + /urix@0.1.0: resolution: {integrity: sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==} deprecated: Please see https://github.com/lydell/urix#deprecated @@ -7714,6 +8334,17 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + dev: false + + /url-template@2.0.8: + resolution: {integrity: sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==} + dev: false + /use@3.1.1: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} @@ -7732,7 +8363,11 @@ packages: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. hasBin: true - dev: true + + /uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -7755,6 +8390,15 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + /verror@1.10.0: + resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} + engines: {'0': node >=0.6.0} + dependencies: + assert-plus: 1.0.0 + core-util-is: 1.0.2 + extsprintf: 1.3.0 + dev: false + /wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} dependencies: