From d1f52a47f794de9bf513e941e1c2dd4988de2b6e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Mar 2023 15:45:21 +0000 Subject: [PATCH 001/232] rtm-server: start a mock service --- packages/rtm-server/notes | 19 ++++ packages/rtm-server/package.json | 30 ++++++ packages/rtm-server/src/api.ts | 3 + packages/rtm-server/src/index.ts | 0 packages/rtm-server/src/mock/index.ts | 126 ++++++++++++++++++++++++++ packages/rtm-server/src/server.ts | 24 +++++ packages/rtm-server/test/mock.test.ts | 104 +++++++++++++++++++++ packages/rtm-server/tsconfig.json | 4 + pnpm-lock.yaml | 79 ++++++++-------- 9 files changed, 349 insertions(+), 40 deletions(-) create mode 100644 packages/rtm-server/notes create mode 100644 packages/rtm-server/package.json create mode 100644 packages/rtm-server/src/api.ts create mode 100644 packages/rtm-server/src/index.ts create mode 100644 packages/rtm-server/src/mock/index.ts create mode 100644 packages/rtm-server/src/server.ts create mode 100644 packages/rtm-server/test/mock.test.ts create mode 100644 packages/rtm-server/tsconfig.json diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes new file mode 100644 index 000000000..3fb2712d6 --- /dev/null +++ b/packages/rtm-server/notes @@ -0,0 +1,19 @@ +lightning must today have a model and execution plan for a workflow + +It needs to know where and when to start (triggers will still be handled by Lightning), and on completion, what to do next + +I'm gonna have to re-implement that in js] + +The more we speak to lightning, a) the tighter the coupling and b) the greater impact of downtime + +right now it looks like we have to: + +- Ask lightning for a workflow id +- Now ask lightning for an execution plan +- On each job, ask lightning for the expression an state + +Just not sure that make sense. + +I think the execution plan needs to be added to the queue and we read this off once. + +Should a runtime manager be responsible for keeping its own history? I dont think so, I think it should be stateless really. Makes scaling easier. diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json new file mode 100644 index 000000000..2d0c24a40 --- /dev/null +++ b/packages/rtm-server/package.json @@ -0,0 +1,30 @@ +{ + "name": "@openfn/rtm-server", + "version": "0.0.1", + "description": "A REST API wrapper around a runtime manager", + "main": "index.js", + "type": "module", + "private": true, + "scripts": { + "test": "pnpm ava", + "test:types": "pnpm tsc --noEmit --project tsconfig.json", + "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", + "build:watch": "pnpm build --watch", + "serve": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/server/index.ts'" + }, + "author": "Open Function Group ", + "license": "ISC", + "dependencies": { + "koa": "^2.13.4" + }, + "devDependencies": { + "@types/koa": "^2.13.5", + "ava": "5.1.0", + "nodemon": "^2.0.19", + "ts-node": "^10.9.1", + "tslib": "^2.4.0", + "tsm": "^2.2.2", + "tsup": "^6.2.3", + "typescript": "^4.6.4" + } +} diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts new file mode 100644 index 000000000..bbf3ec1ed --- /dev/null +++ b/packages/rtm-server/src/api.ts @@ -0,0 +1,3 @@ +// this defines the main API routes in a nice central place + +koa.get('healthcheck', middleware.healthcheck); diff --git a/packages/rtm-server/src/index.ts b/packages/rtm-server/src/index.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/rtm-server/src/mock/index.ts b/packages/rtm-server/src/mock/index.ts new file mode 100644 index 000000000..d794e8abd --- /dev/null +++ b/packages/rtm-server/src/mock/index.ts @@ -0,0 +1,126 @@ +import { EventEmitter } from 'node:events'; + +// A mock runtime manager +// + +// TODO do we mean job, or workflow? +// I think we have both? +type RTMEvent = + | 'job-start' + | 'job-end' + | 'job-log' + | 'job-error' + | 'workflow-start' + | 'workflow-end'; + +type State = any; + +type Workflow = any; + +type Job = { expression: string; state?: State }; + +type FetchWorkflowFn = (workflowId: string) => Promise; + +type FetchJobFn = (jobId: string) => Promise; + +const mockFetchWorkflow = (workflowId: string) => + new Promise((resolve) => resolve({ job: 'job1', next: [] })); + +const mockFetchJob = (jobId: string) => + new Promise((resolve) => + resolve({ + expression: 'export default [s => s];', + state: {}, + }) + ); + +let id = 0; +const getNewJobId = () => ++id; + +// The mock will need some kind of helper function to get from a queue +// This could call out to an endpoint or use something memory +// actually i don't think that's right +function createMock( + fetchWorkflow = mockFetchWorkflow, + fetchJob = mockFetchJob +) { + const mockResults: Record = {}; + + // This at the moment is an aspirational API - its what I think we want + + const bus = new EventEmitter(); + + const dispatch = (type: RTMEvent, args?: any) => { + // TODO add performance metrics to every event? + bus.emit(type, args); + + // TOOD return an unsubscribe API? + }; + + const on = (event: RTMEvent, fn: (evt: any) => void) => { + bus.addListener(event, fn); + }; + + // who handles workflows? + // a) we are given all the data about a workflow at once and we just chew through it + // (may not be possible?) + // b) we call out to get the inputs to each workflow when we start + // But who do we call out to? + // Arch doc says lightning, but I need to decouple this here + // Right now this is a string of job ids + // A workflow isn't just an array btw, it's a graph + // That does actually need thinking through + // There's a jobID, which is what Lightning calls the job + // And there's like an executionID or a runId, which is what the RTM calls the instance run + // These ought to be UUIDs so they're unique across RTMs + const startJob = (jobId: string) => { + const runId = getNewJobId(); + + // Get the job details from lightning + fetchJob(jobId).then(() => { + // start instantly and emit as it goes + dispatch('job-start', { jobId, runId }); + + // TODO random timeout + // What is a job log? Anything emitted by the RTM I guess? + // Namespaced to compile, r/t, job etc. + // It's the json output of the logger + dispatch('job-log', { jobId, runId }); + + // TODO random timeout + const finalState = mockResults.hasOwnProperty(jobId) + ? mockResults[jobId] + : {}; + dispatch('job-end', { jobId, runId, state: finalState }); + }); + + return id; + }; + + const startWorkflow = (jobs: string[]) => { + // Get the execution plan from lightning + dispatch('workflow-start'); + + dispatch('workflow-end'); + }; + + // return a list of jobs in progress + const getStatus = () => {}; + + const _setJobResult = (jobId: string, result: any) => { + mockResults[jobId] = result; + }; + + return { + on, + // TODO runWorkflow? executeWorkflow? + startWorkflow, + startJob, // this is more of an internal API + getStatus, + + // mock APIs + _setJobResult, + }; +} + +export default createMock; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts new file mode 100644 index 000000000..47453cfd1 --- /dev/null +++ b/packages/rtm-server/src/server.ts @@ -0,0 +1,24 @@ +/** + * server needs to + * + * - create a runtime maager + * - know how to speak to a lightning endpoint to fetch workflows + * Is this just a string url? + * + */ + +// This loop will call out to ask for work, with a backof +const workerLoop = () => {}; + +type ServerOptions = { + backoff: number; + maxWorkflows: number; + port: number; +}; + +function createServer(options, rtm) { + // if not rtm create a mock + // setup routes + // start listening on options.port + // return the server +} diff --git a/packages/rtm-server/test/mock.test.ts b/packages/rtm-server/test/mock.test.ts new file mode 100644 index 000000000..9570eda55 --- /dev/null +++ b/packages/rtm-server/test/mock.test.ts @@ -0,0 +1,104 @@ +import test from 'ava'; +import create from '../src/mock'; + +const wait = (fn, maxAttempts = 100) => + new Promise((resolve) => { + let count = 0; + let ival = setInterval(() => { + count++; + if (fn()) { + clearInterval(ival); + resolve(true); + } + + if (count == maxAttempts) { + clearInterval(ival); + resolve(false); + } + }, 100); + }); + +test('It should create a mock runtime manager', (t) => { + const rtm = create(); + const keys = Object.keys(rtm); + t.assert(keys.includes('on')); + t.assert(keys.includes('startWorkflow')); + t.assert(keys.includes('startJob')); + t.assert(keys.includes('getStatus')); +}); + +test('it should dispatch job-start events', async (t) => { + const rtm = create(); + + let evt; + + rtm.on('job-start', (e) => { + evt = e; + }); + + rtm.startJob('a'); + + const didFire = await wait(() => evt); + t.true(didFire); + t.is(evt.jobId, 'a'); + t.truthy(evt.runId); +}); + +test('it should dispatch job-log events', async (t) => { + const rtm = create(); + + let evt; + + rtm.on('job-log', (e) => { + evt = e; + }); + + rtm.startJob('a'); + + const didFire = await wait(() => evt); + + t.true(didFire); + t.is(evt.jobId, 'a'); + t.truthy(evt.runId); +}); + +test('it should dispatch job-end events', async (t) => { + const rtm = create(); + + let evt; + + rtm.on('job-end', (e) => { + evt = e; + }); + + rtm.startJob('a'); + + const didFire = await wait(() => evt); + + t.true(didFire); + t.is(evt.jobId, 'a'); + t.truthy(evt.runId); + t.truthy(evt.state); +}); + +test('it should mock job state', async (t) => { + const rtm = create(); + + const result = 42; + + rtm._setJobResult('a', result); + let evt; + + rtm.on('job-end', (e) => { + evt = e; + }); + + rtm.startJob('a'); + + const didFire = await wait(() => evt); + + t.true(didFire); + t.is(evt.jobId, 'a'); + t.truthy(evt.runId); + t.is(evt.state, result); +}); diff --git a/packages/rtm-server/tsconfig.json b/packages/rtm-server/tsconfig.json new file mode 100644 index 000000000..b3d766fc1 --- /dev/null +++ b/packages/rtm-server/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.common", + "include": ["src/**/*.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 552cdfec5..bde16feb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -324,6 +324,37 @@ importers: specifier: ^4.8.3 version: 4.8.3 + packages/rtm-server: + dependencies: + koa: + specifier: ^2.13.4 + version: 2.13.4 + devDependencies: + '@types/koa': + specifier: ^2.13.5 + version: 2.13.5 + ava: + specifier: 5.1.0 + version: 5.1.0 + nodemon: + specifier: ^2.0.19 + version: 2.0.19 + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@18.15.3)(typescript@4.8.3) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + tsm: + specifier: ^2.2.2 + version: 2.2.2 + tsup: + specifier: ^6.2.3 + version: 6.2.3(ts-node@10.9.1)(typescript@4.8.3) + typescript: + specifier: ^4.6.4 + version: 4.8.3 + packages/runtime: dependencies: '@openfn/logger': @@ -1157,24 +1188,20 @@ packages: resolution: {integrity: sha512-jOdnI/3qTpHABjM5cx1Hc0sKsPoYCp+DP/GJRGtDlPd7fiV9oXGGIcjW/ZOxLIvjGz8MA+uMZI9metHlgqbgwQ==} dependencies: '@types/node': 18.15.3 - dev: false /@types/body-parser@1.19.2: resolution: {integrity: sha512-ALYone6pm6QmwZoAgeyNksccT9Q4AWZQ6PvfwR37GT6r6FWUPguq6sUmNGSMV2Wr761oQoBxwGGa6DR5o1DC9g==} dependencies: '@types/connect': 3.4.35 '@types/node': 18.15.3 - dev: false /@types/connect@3.4.35: resolution: {integrity: sha512-cdeYyv4KWoEgpBISTxWvqYsVy444DOqehiF3fM3ne10AmJ62RSyNkUnxMJXHQWRQQX2eR94m5y1IZyDwBjV9FQ==} dependencies: '@types/node': 18.15.3 - dev: false /@types/content-disposition@0.5.5: resolution: {integrity: sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA==} - dev: false /@types/cookies@0.7.7: resolution: {integrity: sha512-h7BcvPUogWbKCzBR2lY4oqaZbO3jXZksexYJVFvkrFeLgbZjQkU4x8pRq6eg2MHXQhY0McQdqmmsxRWlVAHooA==} @@ -1183,7 +1210,6 @@ packages: '@types/express': 4.17.13 '@types/keygrip': 1.0.2 '@types/node': 18.15.3 - dev: false /@types/events@3.0.0: resolution: {integrity: sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==} @@ -1195,7 +1221,6 @@ packages: '@types/node': 18.15.3 '@types/qs': 6.9.7 '@types/range-parser': 1.2.4 - dev: false /@types/express@4.17.13: resolution: {integrity: sha512-6bSZTPaTIACxn48l50SR+axgrqm6qXFIxrdAKaG6PaJk3+zuUr35hBlgT7vOmJcum+OEaIBLtHV/qloEAFITeA==} @@ -1204,15 +1229,12 @@ packages: '@types/express-serve-static-core': 4.17.30 '@types/qs': 6.9.7 '@types/serve-static': 1.15.0 - dev: false /@types/http-assert@1.5.3: resolution: {integrity: sha512-FyAOrDuQmBi8/or3ns4rwPno7/9tJTijVW6aQQjK02+kOQ8zmoNg2XJtAuQhvQcy1ASJq38wirX5//9J1EqoUA==} - dev: false /@types/http-errors@1.8.2: resolution: {integrity: sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==} - dev: false /@types/is-ci@3.0.0: resolution: {integrity: sha512-Q0Op0hdWbYd1iahB+IFNQcWXFq4O0Q5MwQP7uN0souuQ4rPg1vEYcnIOfr1gY+M+6rc8FGoRaBO1mOOvL29sEQ==} @@ -1230,13 +1252,11 @@ packages: /@types/keygrip@1.0.2: resolution: {integrity: sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==} - dev: false /@types/koa-compose@3.2.5: resolution: {integrity: sha512-B8nG/OoE1ORZqCkBVsup/AKcvjdgoHnfi4pZMn5UwAPCbhk/96xyv284eBYW8JlQbQ7zDmnpFr68I/40mFoIBQ==} dependencies: '@types/koa': 2.13.5 - dev: false /@types/koa@2.13.5: resolution: {integrity: sha512-HSUOdzKz3by4fnqagwthW/1w/yJspTgppyyalPVbgZf8jQWvdIXcVW5h2DGtw4zYntOaeRGx49r1hxoPWrD4aA==} @@ -1249,11 +1269,9 @@ packages: '@types/keygrip': 1.0.2 '@types/koa-compose': 3.2.5 '@types/node': 18.15.3 - dev: false /@types/mime@3.0.1: resolution: {integrity: sha512-Y4XFY5VJAuw0FgAqPNd6NNoV44jbq9Bz2L7Rh/J6jLTiHBSBJa9fxqQIvkIld4GsoDOcCbvzOUAbLPsSKKg+uA==} - dev: false /@types/minimist@1.2.2: resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} @@ -1307,11 +1325,9 @@ packages: /@types/qs@6.9.7: resolution: {integrity: sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw==} - dev: false /@types/range-parser@1.2.4: resolution: {integrity: sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==} - dev: false /@types/react-dom@18.0.6: resolution: {integrity: sha512-/5OFZgfIPSwy+YuIBP/FgJnQnsxhZhjjrnxudMddeblOouIodEQ75X14Rr4wGSG/bknL+Omy9iWlLo1u/9GzAA==} @@ -1348,7 +1364,6 @@ packages: dependencies: '@types/mime': 3.0.1 '@types/node': 18.15.3 - dev: false /@types/workerpool@6.1.0: resolution: {integrity: sha512-C+J/c1BHyc351xJuiH2Jbe+V9hjf5mCzRP0UK4KEpF5SpuU+vJ/FC5GLZsCU/PJpp/3I6Uwtfm3DG7Lmrb7LOQ==} @@ -1423,6 +1438,7 @@ packages: resolution: {integrity: sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==} engines: {node: '>=0.4.0'} hasBin: true + dev: false /acorn@8.8.1: resolution: {integrity: sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==} @@ -1655,7 +1671,7 @@ packages: supertap: 3.0.1 temp-dir: 3.0.0 write-file-atomic: 5.0.0 - yargs: 17.6.2 + yargs: 17.7.2 transitivePeerDependencies: - supports-color @@ -3118,6 +3134,7 @@ packages: glob-parent: 5.1.2 merge2: 1.4.1 micromatch: 4.0.5 + dev: true /fast-glob@3.3.0: resolution: {integrity: sha512-ChDuvbOypPuNjO8yIDf36x7BlZX1smcUMTTcyoIjycexOxd6DFsKsg21qVBzEmr3G7fUKIRy2/psii+CIUt7FA==} @@ -3128,7 +3145,6 @@ packages: glob-parent: 5.1.2 merge2: 1.4.1 micromatch: 4.0.5 - dev: true /fast-json-patch@3.1.1: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} @@ -3420,8 +3436,8 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dependencies: dir-glob: 3.0.1 - fast-glob: 3.2.12 - ignore: 5.2.0 + fast-glob: 3.3.0 + ignore: 5.2.4 merge2: 1.4.1 slash: 4.0.0 @@ -3605,14 +3621,9 @@ packages: resolution: {integrity: sha512-yiWd4GVmJp0Q6ghmM2B/V3oZGRmjrKLXvHR3TE1nfoXsmoggllfZUQe74EN0fJdPFZu2NIvNdrMMLm3OsV7Ohw==} engines: {node: '>=10 <11 || >=12 <13 || >=14'} - /ignore@5.2.0: - resolution: {integrity: sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==} - engines: {node: '>= 4'} - /ignore@5.2.4: resolution: {integrity: sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ==} engines: {node: '>= 4'} - dev: true /imurmurhash@0.1.4: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} @@ -5937,7 +5948,7 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 '@types/node': 17.0.45 - acorn: 8.8.0 + acorn: 8.8.1 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 @@ -5968,7 +5979,7 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 '@types/node': 17.0.45 - acorn: 8.8.0 + acorn: 8.8.1 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 @@ -5999,7 +6010,7 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 '@types/node': 18.15.3 - acorn: 8.8.0 + acorn: 8.8.1 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 @@ -6480,18 +6491,6 @@ packages: yargs-parser: 18.1.3 dev: true - /yargs@17.6.2: - resolution: {integrity: sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==} - engines: {node: '>=12'} - dependencies: - cliui: 8.0.1 - escalade: 3.1.1 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - string-width: 4.2.3 - y18n: 5.0.8 - yargs-parser: 21.1.1 - /yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} From bc66c24324bb35ab89567df90cef11a9ea77c48a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Mar 2023 17:34:55 +0000 Subject: [PATCH 002/232] rtm-server: sort of add integration tests --- packages/rtm-server/package.json | 5 +- packages/rtm-server/src/api.ts | 15 +++++- packages/rtm-server/src/index.ts | 3 ++ .../rtm-server/src/middleware/healthcheck.ts | 4 ++ packages/rtm-server/src/mock/lightning.ts | 43 +++++++++++++++++ .../src/mock/{index.ts => runtime-manager.ts} | 7 +-- packages/rtm-server/src/server.ts | 46 ++++++++++++++++++- packages/rtm-server/src/start.ts | 0 packages/rtm-server/test/integration.test.ts | 35 ++++++++++++++ .../runtime-manager.test.ts} | 21 ++------- packages/rtm-server/test/server.test.ts | 17 +++++++ packages/rtm-server/test/util.ts | 16 +++++++ 12 files changed, 187 insertions(+), 25 deletions(-) create mode 100644 packages/rtm-server/src/middleware/healthcheck.ts create mode 100644 packages/rtm-server/src/mock/lightning.ts rename packages/rtm-server/src/mock/{index.ts => runtime-manager.ts} (94%) create mode 100644 packages/rtm-server/src/start.ts create mode 100644 packages/rtm-server/test/integration.test.ts rename packages/rtm-server/test/{mock.test.ts => mock/runtime-manager.test.ts} (80%) create mode 100644 packages/rtm-server/test/server.test.ts create mode 100644 packages/rtm-server/test/util.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 2d0c24a40..99941744d 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -10,15 +10,18 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", - "serve": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/server/index.ts'" + "start": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/index.ts'" }, "author": "Open Function Group ", "license": "ISC", "dependencies": { + "@koa/router": "^12.0.0", + "axios": "^1.3.4", "koa": "^2.13.4" }, "devDependencies": { "@types/koa": "^2.13.5", + "@types/node": "^18.15.3", "ava": "5.1.0", "nodemon": "^2.0.19", "ts-node": "^10.9.1", diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts index bbf3ec1ed..021a5694d 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api.ts @@ -1,3 +1,16 @@ +import Router from '@koa/router'; + +import healthcheck from './middleware/healthcheck'; // this defines the main API routes in a nice central place -koa.get('healthcheck', middleware.healthcheck); +// So what is the API of this server? +// It's mostly a pull model, apart from I think the healthcheck +const createAPI = () => { + const router = new Router(); + + router.get('/healthcheck', healthcheck); + + return router; +}; + +export default createAPI; diff --git a/packages/rtm-server/src/index.ts b/packages/rtm-server/src/index.ts index e69de29bb..2e5d5b278 100644 --- a/packages/rtm-server/src/index.ts +++ b/packages/rtm-server/src/index.ts @@ -0,0 +1,3 @@ +import createServer from './server'; + +const server = createServer(); diff --git a/packages/rtm-server/src/middleware/healthcheck.ts b/packages/rtm-server/src/middleware/healthcheck.ts new file mode 100644 index 000000000..658dc7564 --- /dev/null +++ b/packages/rtm-server/src/middleware/healthcheck.ts @@ -0,0 +1,4 @@ +export default (ctx) => { + ctx.status = 200; + ctx.body = 'OK'; +}; diff --git a/packages/rtm-server/src/mock/lightning.ts b/packages/rtm-server/src/mock/lightning.ts new file mode 100644 index 000000000..ea73e9e77 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning.ts @@ -0,0 +1,43 @@ +import Koa from 'koa'; +import Router from '@koa/router'; + +// a mock lightning server +const createLightningServer = (options = {}) => { + const app = new Koa(); + + const queue: string[] = []; + const workflows = {}; + const jobs = {}; + + const router = new Router(); + + router.get('/workflow/:id', () => {}); + + router.get('/job/:id', () => {}); + + router.get('/queue', (ctx) => { + console.log('***'); + const first = queue.shift(); + if (first) { + console.log(first); + ctx.body = JSON.stringify({ workflowId: first }); + } else { + ctx.body = undefined; + } + ctx.status = 200; + }); + + app.addToQueue = (workflowId: string) => { + console.log('add to queue', workflowId); + queue.push(workflowId); + }; + + app.use(router.routes()); + app.use(router.allowedMethods()); + + app.listen(options.port || 8888); + + return app; +}; + +export default createLightningServer; diff --git a/packages/rtm-server/src/mock/index.ts b/packages/rtm-server/src/mock/runtime-manager.ts similarity index 94% rename from packages/rtm-server/src/mock/index.ts rename to packages/rtm-server/src/mock/runtime-manager.ts index d794e8abd..73522a518 100644 --- a/packages/rtm-server/src/mock/index.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -97,11 +97,12 @@ function createMock( return id; }; - const startWorkflow = (jobs: string[]) => { + const startWorkflow = (workflowId: string) => { + // console.log('start-workflow', workflowId); // Get the execution plan from lightning - dispatch('workflow-start'); + dispatch('workflow-start', { workflowId }); - dispatch('workflow-end'); + dispatch('workflow-end', { workflowId }); }; // return a list of jobs in progress diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 47453cfd1..98e8fe505 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -7,18 +7,60 @@ * */ +import Koa from 'koa'; +import axios from 'axios'; +import createAPI from './api'; + +import createMockRTM from './mock/runtime-manager'; + // This loop will call out to ask for work, with a backof -const workerLoop = () => {}; +const workerLoop = async (url: string, rtm: any) => { + let timeout = 100; // TODO strange stuff happens if this has a low value + console.log(`${url}/queue`); + const result = await axios.get(`${url}/queue`); + if (result.data) { + console.log(result.data); + rtm.startWorkflow(result.data.workflowId); + } else { + timeout = timeout * 2; + } + setTimeout(() => { + workerLoop(url, rtm); + }, timeout); +}; type ServerOptions = { backoff: number; maxWorkflows: number; port: number; + lightning: string; // url to lightning isnstance + rtm?: any; }; -function createServer(options, rtm) { +function createServer(options = {}) { + const app = new Koa(); + + const rtm = options.rtm || new createMockRTM(); + + const apiRouter = createAPI(); + app.use(apiRouter.routes()); + app.use(apiRouter.allowedMethods()); + + app.listen(options.port || 1234); + + if (options.lightning) { + workerLoop(options.lightning, rtm); + } + + // TMP doing this for tests but maybe its better done externally + app.on = (...args) => rtm.on(...args); + + return app; + // if not rtm create a mock // setup routes // start listening on options.port // return the server } + +export default createServer; diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts new file mode 100644 index 000000000..0bd3b7dd6 --- /dev/null +++ b/packages/rtm-server/test/integration.test.ts @@ -0,0 +1,35 @@ +// This will test both servers talking to each other +import test from 'ava'; +import axios from 'axios'; +import createRTMServer from '../src/server'; +import createLightningServer from '../src/mock/lightning'; + +import { wait } from './util'; + +let lng; +let rtm; + +const urls = { + rtm: 'http://localhost:7777', + lng: 'http://localhost:8888', +}; + +test.before(() => { + lng = createLightningServer({ port: 8888 }); + rtm = createRTMServer({ port: 7777, lightning: urls.lng }); +}); + +// TODO get this working +test.serial('should pick up a workflow in the queue', async (t) => { + let found: false | string = false; + + rtm.on('workflow-start', (e) => { + // console.log(e); + found = e.workflowId; + }); + + lng.addToQueue('a'); + + await wait(() => found); + t.is(found as unknown as string, 'a'); +}); diff --git a/packages/rtm-server/test/mock.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts similarity index 80% rename from packages/rtm-server/test/mock.test.ts rename to packages/rtm-server/test/mock/runtime-manager.test.ts index 9570eda55..eb2f636b2 100644 --- a/packages/rtm-server/test/mock.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -1,22 +1,7 @@ import test from 'ava'; -import create from '../src/mock'; - -const wait = (fn, maxAttempts = 100) => - new Promise((resolve) => { - let count = 0; - let ival = setInterval(() => { - count++; - if (fn()) { - clearInterval(ival); - resolve(true); - } - - if (count == maxAttempts) { - clearInterval(ival); - resolve(false); - } - }, 100); - }); +import create from '../../src/mock/runtime-manager'; + +import { wait } from '../util'; test('It should create a mock runtime manager', (t) => { const rtm = create(); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts new file mode 100644 index 000000000..a7b18dbae --- /dev/null +++ b/packages/rtm-server/test/server.test.ts @@ -0,0 +1,17 @@ +import test from 'ava'; +import axios from 'axios'; +import createServer from '../src/server'; + +let server; + +const url = 'http://localhost:7777'; + +test.beforeEach(() => { + server = createServer({ port: 7777 }); +}); + +test('healthcheck', async (t) => { + const result = await axios.get(`${url}/healthcheck`); + t.is(result.status, 200); + t.is(result.data, 'OK'); +}); diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts new file mode 100644 index 000000000..aac8ae618 --- /dev/null +++ b/packages/rtm-server/test/util.ts @@ -0,0 +1,16 @@ +export const wait = (fn, maxAttempts = 100) => + new Promise((resolve) => { + let count = 0; + let ival = setInterval(() => { + count++; + if (fn()) { + clearInterval(ival); + resolve(true); + } + + if (count == maxAttempts) { + clearInterval(ival); + resolve(false); + } + }, 100); + }); From 950c33aed3812063d62f889ee5e9d5e08dd133f9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Mar 2023 09:18:48 +0000 Subject: [PATCH 003/232] package lock --- pnpm-lock.yaml | 52 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bde16feb6..b45f04ec0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -326,6 +326,12 @@ importers: packages/rtm-server: dependencies: + '@koa/router': + specifier: ^12.0.0 + version: 12.0.0 + axios: + specifier: ^1.3.4 + version: 1.3.4 koa: specifier: ^2.13.4 version: 2.13.4 @@ -333,6 +339,9 @@ importers: '@types/koa': specifier: ^2.13.5 version: 2.13.5 + '@types/node': + specifier: ^18.15.3 + version: 18.15.3 ava: specifier: 5.1.0 version: 5.1.0 @@ -1110,6 +1119,16 @@ packages: '@jridgewell/resolve-uri': 3.1.0 '@jridgewell/sourcemap-codec': 1.4.14 + /@koa/router@12.0.0: + resolution: {integrity: sha512-cnnxeKHXlt7XARJptflGURdJaO+ITpNkOHmQu7NHmCoRinPbyvFzce/EG/E8Zy81yQ1W9MoSdtklc3nyaDReUw==} + engines: {node: '>= 12'} + dependencies: + http-errors: 2.0.0 + koa-compose: 4.1.0 + methods: 1.1.2 + path-to-regexp: 6.2.1 + dev: false + /@manypkg/find-root@1.1.0: resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} dependencies: @@ -1305,10 +1324,6 @@ packages: /@types/node@18.15.3: resolution: {integrity: sha512-p6ua9zBxz5otCmbpb5D3U4B5Nanw6Pk3PPyX05xnxbB/fRv71N7CPmORg7uAD5P70T0xmx1pzAx/FUfa5X+3cw==} - /@types/node@18.7.14: - resolution: {integrity: sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==} - dev: false - /@types/node@18.7.18: resolution: {integrity: sha512-m+6nTEOadJZuTPkKR/SYK3A2d7FZrgElol9UP1Kae90VVU4a6mxnPuLiIW1m4Cq4gZ/nWb9GrdVXJCoCazDAbg==} @@ -1368,7 +1383,7 @@ packages: /@types/workerpool@6.1.0: resolution: {integrity: sha512-C+J/c1BHyc351xJuiH2Jbe+V9hjf5mCzRP0UK4KEpF5SpuU+vJ/FC5GLZsCU/PJpp/3I6Uwtfm3DG7Lmrb7LOQ==} dependencies: - '@types/node': 18.7.14 + '@types/node': 18.15.3 dev: false /@types/wrap-ansi@3.0.0: @@ -1609,7 +1624,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -1685,6 +1699,16 @@ packages: - debug dev: true + /axios@1.3.4: + resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} + dependencies: + follow-redirects: 1.15.2 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + dev: false + /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} dev: true @@ -2068,7 +2092,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -2343,7 +2366,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3259,7 +3281,6 @@ packages: peerDependenciesMeta: debug: optional: true - dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -3273,7 +3294,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -3588,7 +3608,6 @@ packages: setprototypeof: 1.2.0 statuses: 2.0.1 toidentifier: 1.0.1 - dev: true /http-parser-js@0.5.8: resolution: {integrity: sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==} @@ -4290,6 +4309,11 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} + /methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + dev: false + /micromatch@3.1.10: resolution: {integrity: sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==} engines: {node: '>=0.10.0'} @@ -4782,6 +4806,10 @@ packages: resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} dev: true + /path-to-regexp@6.2.1: + resolution: {integrity: sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==} + dev: false + /path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -4990,7 +5018,6 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -5589,7 +5616,6 @@ packages: /statuses@2.0.1: resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} engines: {node: '>= 0.8'} - dev: true /stream-combiner@0.0.4: resolution: {integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==} From 427e9726d805c543885acbc4fc96d4c9b5a5c3a6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 17 Mar 2023 16:57:27 +0000 Subject: [PATCH 004/232] rtm-server: String togeether a mock lightning server --- packages/rtm-server/package.json | 3 +- packages/rtm-server/src/mock/lightning.ts | 127 ++++++++++++++++-- .../rtm-server/src/mock/runtime-manager.ts | 10 +- packages/rtm-server/src/server.ts | 2 +- packages/rtm-server/src/start.ts | 0 packages/rtm-server/src/types.d.ts | 44 ++++++ .../rtm-server/test/mock/lightning.test.ts | 115 ++++++++++++++++ pnpm-lock.yaml | 58 +++++++- 8 files changed, 333 insertions(+), 26 deletions(-) delete mode 100644 packages/rtm-server/src/start.ts create mode 100644 packages/rtm-server/src/types.d.ts create mode 100644 packages/rtm-server/test/mock/lightning.test.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 99941744d..b69b2b546 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -17,7 +17,8 @@ "dependencies": { "@koa/router": "^12.0.0", "axios": "^1.3.4", - "koa": "^2.13.4" + "koa": "^2.13.4", + "koa-bodyparser": "^4.4.0" }, "devDependencies": { "@types/koa": "^2.13.5", diff --git a/packages/rtm-server/src/mock/lightning.ts b/packages/rtm-server/src/mock/lightning.ts index ea73e9e77..3e0987e78 100644 --- a/packages/rtm-server/src/mock/lightning.ts +++ b/packages/rtm-server/src/mock/lightning.ts @@ -1,40 +1,147 @@ +import { EventEmitter } from 'node:events'; import Koa from 'koa'; +import bodyParser from 'koa-bodyparser'; import Router from '@koa/router'; +import { Workflow, JobNode } from '../types'; +import { RTMEvent } from './runtime-manager'; + +/* +Expected server API +- GET workflow +- GET job +- GET queue +- POST notify +*/ + +const workflows = { + 'workflow-1': { + id: 'workflow-1', + state: {}, + start: 'job-1', + plan: [ + { + id: 'job-1', + // no expression in the plan + }, + ], + }, + 'workflow-2': { + id: 'workflow-1', + state: {}, + start: 'job-1', + plan: [ + { + id: 'job-1', + upstream: 'job-2', + }, + { + id: 'job-2', + }, + ], + }, +}; + +const jobs = { + 'job-1': { + id: 'job-1', + expression: 'export default [s => s];', + }, + 'job-2': { + id: 'job-2', + expression: 'export default [s => s];', + }, +}; + +type NotifyEvent = { + event: RTMEvent; + workflow: string; // workflow id + [key: string]: any; +}; + // a mock lightning server const createLightningServer = (options = {}) => { const app = new Koa(); const queue: string[] = []; - const workflows = {}; - const jobs = {}; const router = new Router(); - router.get('/workflow/:id', () => {}); + const events = new EventEmitter(); - router.get('/job/:id', () => {}); + // GET Workflow: + // 200 - workflow json as body + // 404 - workflow not found. No body. + router.get('/workflow/:id', (ctx) => { + const { id } = ctx.params; + if (workflows[id]) { + ctx.status = 200; + ctx.body = JSON.stringify(workflows[id]); + } else { + ctx.status = 404; + } + }); + // GET Job: + // 200 - job json as body + // 404 - job not found. No body. + router.get('/job/:id', (ctx) => { + const { id } = ctx.params; + if (jobs[id]) { + ctx.status = 200; + ctx.body = JSON.stringify(jobs[id]); + } else { + ctx.status = 404; + } + }); + + // TODO I think this is just GET workflow? Or get work? + // GET queue: + // 200 - returned a queue item (json in body) + // 204 - queue empty (no body) router.get('/queue', (ctx) => { - console.log('***'); const first = queue.shift(); if (first) { - console.log(first); ctx.body = JSON.stringify({ workflowId: first }); + ctx.status = 200; } else { ctx.body = undefined; + ctx.status = 204; } - ctx.status = 200; }); + // Notify of some job update + // proxy to event emitter + // { event: 'event-name', workflow: 'workflow-id' } + // TODO cty.body is always undefined ??? + router.post('/notify', (ctx) => { + const evt = ctx.data as NotifyEvent; + // console.log(ctx); + // TODO pull out the payload + events.emit('notify', evt); + + ctx.status = 202; + }); + + // Dev APIs for unit testing + app.addWorkflow = (workflow: Workflow) => { + workflows[workflow.id] = workflow; + }; + app.addJob = (job: JobNode) => { + jobs[job.id] = job; + }; app.addToQueue = (workflowId: string) => { - console.log('add to queue', workflowId); queue.push(workflowId); }; + app.on = (event: 'notify', fn: (evt: any) => void) => { + events.addListener(event, fn); + }; + app.once = (event: 'notify', fn: (evt: any) => void) => { + events.once(event, fn); + }; + app.use(bodyParser()); app.use(router.routes()); - app.use(router.allowedMethods()); - app.listen(options.port || 8888); return app; diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 73522a518..e5eeb2ed7 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -1,11 +1,13 @@ import { EventEmitter } from 'node:events'; +import type { State, Job, Workflow } from '../types'; + // A mock runtime manager // // TODO do we mean job, or workflow? // I think we have both? -type RTMEvent = +export type RTMEvent = | 'job-start' | 'job-end' | 'job-log' @@ -13,12 +15,6 @@ type RTMEvent = | 'workflow-start' | 'workflow-end'; -type State = any; - -type Workflow = any; - -type Job = { expression: string; state?: State }; - type FetchWorkflowFn = (workflowId: string) => Promise; type FetchJobFn = (jobId: string) => Promise; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 98e8fe505..0124a9156 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -33,7 +33,7 @@ type ServerOptions = { backoff: number; maxWorkflows: number; port: number; - lightning: string; // url to lightning isnstance + lightning: string; // url to lightning instance rtm?: any; }; diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts new file mode 100644 index 000000000..fb6648b1d --- /dev/null +++ b/packages/rtm-server/src/types.d.ts @@ -0,0 +1,44 @@ +export type State = any; + +type JobID = string; + +// This job structure lets us lazily evalulate the expression quite nicely +export type JobNode = { + id: string; // the ID is a sha of the code + adaptor: string; + + expression?: string; // the code we actually want to execute. Could be lazy loaded + state?: State; // should state be part of a job? Not really? + + upstream: + | JobNode // shorthand for { default } + | { + success: JobNode; + error: JobNode; + default: JobNode; + }; +}; + +export type Workflow = { + id: string; // uuid + name?: string; // I don't think the RTM cares about this? + + // This is the initial state + state: State | string; // inline obj or UUID (Or sha, whatever) + + // Which job do we start at? + // This isn't neccessarily the first node + start: JobID; + + // This is the execution plan for the workflow + plan: JobNode[]; +}; + +/* + +We can fetch workflow by id +The workflow is a json object +Jobs and probably state objects may be lazily resolved + +A workflow starts at the start and executes jobs until there are no more upstream jobs +*/ diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts new file mode 100644 index 000000000..d2a06ae86 --- /dev/null +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -0,0 +1,115 @@ +import test from 'ava'; +import axios from 'axios'; + +import createLightningServer from '../../src/mock/lightning'; +import { wait } from '../util'; + +const baseUrl = 'http://localhost:8888'; + +let server = createLightningServer({ port: 8888 }); + +const get = (path: string) => axios.get(`${baseUrl}/${path}`); +const post = (path: string, data: any) => + axios.post(`${baseUrl}/${path}`, data); + +test.serial('GET /job - return 404 if no job', async (t) => { + try { + await get('job/x'); + } catch (e) { + t.is(e.response.status, 404); + } +}); + +test.serial('GET /job - return a job', async (t) => { + const { status, data: job } = await get('job/job-1'); + t.is(status, 200); + + t.is(job.id, 'job-1'); + t.truthy(job.expression); +}); + +test.serial('GET /job - return a new mock job', async (t) => { + server.addJob({ id: 'jam' }); + const { status, data: job } = await get('job/jam'); + t.is(status, 200); + + t.is(job.id, 'jam'); +}); + +test.serial('GET /workflow - return 404 if no workflow', async (t) => { + try { + await get('workflow/x'); + } catch (e) { + t.is(e.response.status, 404); + } +}); + +test.serial('GET /workflow - return a workflow', async (t) => { + const { status, data: workflow } = await get('workflow/workflow-1'); + t.is(status, 200); + + t.is(workflow.id, 'workflow-1'); +}); + +test.serial('GET /workflow - return a new mock workflow', async (t) => { + server.addWorkflow({ id: 'jam' }); + const { status, data: workflow } = await get('workflow/jam'); + t.is(status, 200); + + t.is(workflow.id, 'jam'); +}); + +test.serial('GET /queue - return 204 for an empty queue', async (t) => { + const { status, data } = await get('queue'); + t.is(status, 204); + + t.falsy(data); +}); + +test.serial('GET /queue - return 200 with a workflow id', async (t) => { + server.addToQueue('workflow-1'); + const { status, data } = await get('queue'); + t.is(status, 200); + + t.truthy(data); + t.is(data.workflowId, 'workflow-1'); +}); + +test.serial('GET /queue - clear the queue after a request', async (t) => { + server.addToQueue('workflow-1'); + const req1 = await get('queue'); + t.is(req1.status, 200); + + t.is(req1.data.workflowId, 'workflow-1'); + + const req2 = await get('queue'); + t.is(req2.status, 204); + t.falsy(req2.data); +}); + +test.serial('POST /notify - should return 202', async (t) => { + const { status } = await post('notify', {}); + t.is(status, 202); +}); + +test.serial('POST /notify - should echo to event emitter', async (t) => { + let evt; + let didCall = false; + //server.once('notify', (e) => (evt = e)); + server.once('notify', (e) => { + didCall = true; + evt = e; + }); + + const { status } = await post('notify', { + event: 'job-start', + workflowId: 'workflow-1', + }); + t.is(status, 202); + t.true(didCall); + // await wait(() => evt); + + // t.truthy(evt); + // t.is(evt.workflowId, 'job-start'); + // t.is(evt.workflowId, 'workflow-1'); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b45f04ec0..5f9f93483 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -335,6 +335,9 @@ importers: koa: specifier: ^2.13.4 version: 2.13.4 + koa-bodyparser: + specifier: ^4.4.0 + version: 4.4.0 devDependencies: '@types/koa': specifier: ^2.13.5 @@ -1846,6 +1849,11 @@ packages: load-tsconfig: 0.2.5 dev: true + /bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + dev: false + /cac@6.7.14: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} @@ -1879,7 +1887,6 @@ packages: dependencies: function-bind: 1.1.1 get-intrinsic: 1.1.3 - dev: true /callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} @@ -2045,6 +2052,15 @@ packages: engines: {node: '>=0.8'} dev: true + /co-body@6.1.0: + resolution: {integrity: sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==} + dependencies: + inflation: 2.0.0 + qs: 6.11.2 + raw-body: 2.5.2 + type-is: 1.6.18 + dev: false + /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -2162,6 +2178,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /copy-to@2.0.1: + resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} + dev: false + /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -3352,7 +3372,6 @@ packages: /function-bind@1.1.1: resolution: {integrity: sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==} - dev: true /function.prototype.name@1.1.5: resolution: {integrity: sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==} @@ -3378,7 +3397,6 @@ packages: function-bind: 1.1.1 has: 1.0.3 has-symbols: 1.0.3 - dev: true /get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} @@ -3550,7 +3568,6 @@ packages: engines: {node: '>= 0.4.0'} dependencies: function-bind: 1.1.1 - dev: true /heap@0.2.7: resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} @@ -3657,6 +3674,11 @@ packages: resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} engines: {node: '>=12'} + /inflation@2.0.0: + resolution: {integrity: sha512-m3xv4hJYR2oXw4o4Y5l6P5P16WYmazYof+el6Al3f+YlggGj6qT9kImBAnzDelRALnP5d3h4jGBPKzYCizjZZw==} + engines: {node: '>= 0.8.0'} + dev: false + /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} dependencies: @@ -4068,6 +4090,14 @@ packages: engines: {node: '>=6'} dev: true + /koa-bodyparser@4.4.0: + resolution: {integrity: sha512-AXPY7wwKZUmbgb8VkTEUFoRNOlx6aWRJwEnQD+zfNf33/7KSAkN4Oo9BqlIk80D+5TvuqlhpQT5dPVcyxl5Zsw==} + engines: {node: '>=8.0.0'} + dependencies: + co-body: 6.1.0 + copy-to: 2.0.1 + dev: false + /koa-compose@4.1.0: resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} dev: false @@ -4579,7 +4609,6 @@ packages: /object-inspect@1.12.2: resolution: {integrity: sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==} - dev: true /object-keys@1.1.1: resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} @@ -5052,6 +5081,13 @@ packages: engines: {node: '>=6'} dev: true + /qs@6.11.2: + resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.4 + dev: false + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -5074,6 +5110,16 @@ packages: engines: {node: '>= 0.6'} dev: true + /raw-body@2.5.2: + resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} + engines: {node: '>= 0.8'} + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + dev: false + /react-dom@18.2.0(react@18.2.0): resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} peerDependencies: @@ -5431,7 +5477,6 @@ packages: call-bind: 1.0.2 get-intrinsic: 1.1.3 object-inspect: 1.12.2 - dev: true /signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} @@ -6268,7 +6313,6 @@ packages: /unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} - dev: true /unset-value@1.0.0: resolution: {integrity: sha512-PcA2tsuGSF9cnySLHTLSh2qrQiJ70mn+r+Glzxv2TWZblxsxCC52BDlZoPCsz7STd9pN7EZetkWZBAvk4cgZdQ==} From 0386110536f7542c0669795be2717a009e23e1d5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 23 Mar 2023 16:39:03 +0000 Subject: [PATCH 005/232] rtm: big refactor of lightning api --- packages/rtm-server/src/mock/data.ts | 22 ++ packages/rtm-server/src/mock/lightning.ts | 196 +++++++++--------- packages/rtm-server/src/types.d.ts | 67 +++--- .../rtm-server/test/mock/lightning.test.ts | 183 +++++++++------- 4 files changed, 260 insertions(+), 208 deletions(-) create mode 100644 packages/rtm-server/src/mock/data.ts diff --git a/packages/rtm-server/src/mock/data.ts b/packages/rtm-server/src/mock/data.ts new file mode 100644 index 000000000..f0172a81d --- /dev/null +++ b/packages/rtm-server/src/mock/data.ts @@ -0,0 +1,22 @@ +export const credentials = () => ({ + a: { + user: 'bobby', + password: 'password1', + }, +}); + +export const attempts = () => ({ + 'attempt-1': { + id: 'attempt-1', + input: { + data: {}, + }, + plan: [ + { + adaptor: '@openfn/language-common@1.0.0', + expression: 'fn(a => a)', + credential: 'a', + }, + ], + }, +}); diff --git a/packages/rtm-server/src/mock/lightning.ts b/packages/rtm-server/src/mock/lightning.ts index 3e0987e78..e6e77d72c 100644 --- a/packages/rtm-server/src/mock/lightning.ts +++ b/packages/rtm-server/src/mock/lightning.ts @@ -3,54 +3,12 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import Router from '@koa/router'; -import { Workflow, JobNode } from '../types'; +import * as data from './data'; +import { LightningAttempt, State } from '../types'; import { RTMEvent } from './runtime-manager'; -/* -Expected server API -- GET workflow -- GET job -- GET queue -- POST notify -*/ - -const workflows = { - 'workflow-1': { - id: 'workflow-1', - state: {}, - start: 'job-1', - plan: [ - { - id: 'job-1', - // no expression in the plan - }, - ], - }, - 'workflow-2': { - id: 'workflow-1', - state: {}, - start: 'job-1', - plan: [ - { - id: 'job-1', - upstream: 'job-2', - }, - { - id: 'job-2', - }, - ], - }, -}; - -const jobs = { - 'job-1': { - id: 'job-1', - expression: 'export default [s => s];', - }, - 'job-2': { - id: 'job-2', - expression: 'export default [s => s];', - }, +const unimplemented = (ctx) => { + ctx.statusCode = 501; }; type NotifyEvent = { @@ -59,80 +17,124 @@ type NotifyEvent = { [key: string]: any; }; +export const API_PREFIX = `/api/v1`; + // a mock lightning server const createLightningServer = (options = {}) => { - const app = new Koa(); - + // App state + const credentials = data.credentials(); + const attempts = data.attempts(); const queue: string[] = []; + const results = {}; + // Server setup + const app = new Koa(); const router = new Router(); - const events = new EventEmitter(); + app.use(bodyParser()); - // GET Workflow: - // 200 - workflow json as body - // 404 - workflow not found. No body. - router.get('/workflow/:id', (ctx) => { - const { id } = ctx.params; - if (workflows[id]) { - ctx.status = 200; - ctx.body = JSON.stringify(workflows[id]); - } else { - ctx.status = 404; + // Mock API endpoints + + // POST attempts/next: + // 200 - return an array of pending attempts + // 204 - queue empty (no body) + router.post(`${API_PREFIX}/attempts/next`, (ctx) => { + const { body } = ctx.request; + if (!body || !body.id) { + ctx.status = 400; + return; + } + + let count = ctx.request.query.count || 1; + const payload = []; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + payload.push(queue.shift()); + count -= 1; } - }); - // GET Job: - // 200 - job json as body - // 404 - job not found. No body. - router.get('/job/:id', (ctx) => { - const { id } = ctx.params; - if (jobs[id]) { + if (payload.length > 0) { + ctx.body = JSON.stringify(payload); ctx.status = 200; - ctx.body = JSON.stringify(jobs[id]); } else { - ctx.status = 404; + ctx.body = undefined; + ctx.status = 204; } }); - // TODO I think this is just GET workflow? Or get work? - // GET queue: - // 200 - returned a queue item (json in body) - // 204 - queue empty (no body) - router.get('/queue', (ctx) => { - const first = queue.shift(); - if (first) { - ctx.body = JSON.stringify({ workflowId: first }); + // GET credential/:id + // 200 - return a credential object + // 404 - credential not found + router.get(`${API_PREFIX}/credential/:id`, (ctx) => { + const cred = credentials[ctx.params.id]; + if (cred) { + ctx.body = JSON.stringify(cred); ctx.status = 200; } else { - ctx.body = undefined; - ctx.status = 204; + ctx.status = 404; } }); // Notify of some job update // proxy to event emitter - // { event: 'event-name', workflow: 'workflow-id' } - // TODO cty.body is always undefined ??? - router.post('/notify', (ctx) => { - const evt = ctx.data as NotifyEvent; - // console.log(ctx); - // TODO pull out the payload - events.emit('notify', evt); - - ctx.status = 202; + // { event: 'event-name', ...data } + router.post(`${API_PREFIX}/attempts/notify/:id`, (ctx) => { + const { event: name, ...payload } = ctx.request.body; + + const event = { + id: ctx.params.id, + name, + ...payload, // spread payload? + }; + + events.emit('notify', event); + + ctx.status = 200; }); + // Notify an attempt has finished + // Could be error or success state + // Error or state in payload + // { data } | { error } + router.post(`${API_PREFIX}/attempts/complete/:id`, (ctx) => { + const finalState = ctx.data as State; + + results[ctx.params.id] = finalState; + + ctx.status = 200; + }); + + // Unimplemented API endpoints + + router.get(`${API_PREFIX}/attempts/:id`, unimplemented); + router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 + router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid + + router.get(`${API_PREFIX}/credential/:id`, unimplemented); + router.get(`${API_PREFIX}/attempts/active`, unimplemented); + + router.get(`${API_PREFIX}/workflows`, unimplemented); + router.get(`${API_PREFIX}/workflows/:id`, unimplemented); + + app.use(router.routes()); + // Dev APIs for unit testing - app.addWorkflow = (workflow: Workflow) => { - workflows[workflow.id] = workflow; + app.addCredential = (id: string, cred: Credential) => { + credentials[id] = cred; }; - app.addJob = (job: JobNode) => { - jobs[job.id] = job; + app.addAttempt = (attempt: LightningAttempt) => { + attempts[attempt.id] = attempt; }; - app.addToQueue = (workflowId: string) => { - queue.push(workflowId); + app.addToQueue = (attemptId: string) => { + if (attempts[attemptId]) { + queue.push(attempts[attemptId]); + return true; + } + return false; }; + app.getQueueLength = () => queue.length; app.on = (event: 'notify', fn: (evt: any) => void) => { events.addListener(event, fn); }; @@ -140,9 +142,11 @@ const createLightningServer = (options = {}) => { events.once(event, fn); }; - app.use(bodyParser()); - app.use(router.routes()); - app.listen(options.port || 8888); + const server = app.listen(options.port || 8888); + + app.destroy = () => { + server.close(); + }; return app; }; diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index fb6648b1d..441e5313b 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -1,44 +1,39 @@ -export type State = any; +export type Credential = object; + +export type State = { + data: { + [key: string]: any; + }; + configuration: { + [key: string]: any; + }; + + // technically there should be nothing here + [key: string]: any; +}; -type JobID = string; +// An attempt object returned by Lightning +// Lightning will build this wfrom a Workflow and Attempt +export type LightningAttempt = { + id: string; + input: Omit; // initial state + plan: LightningJob; -// This job structure lets us lazily evalulate the expression quite nicely -export type JobNode = { - id: string; // the ID is a sha of the code - adaptor: string; + // these will probably be included by lightning but we don't care here + status: string; + worker: string; +}; - expression?: string; // the code we actually want to execute. Could be lazy loaded - state?: State; // should state be part of a job? Not really? +export type LightningJob = { + adaptor: string; + expression: string; // the code we actually want to execute. Could be lazy loaded + credential: id; upstream: - | JobNode // shorthand for { default } + | LightningJob // shorthand for { default } | { - success: JobNode; - error: JobNode; - default: JobNode; + success: LightningJob; + error: LightningJob; + default: LightningJob; }; }; - -export type Workflow = { - id: string; // uuid - name?: string; // I don't think the RTM cares about this? - - // This is the initial state - state: State | string; // inline obj or UUID (Or sha, whatever) - - // Which job do we start at? - // This isn't neccessarily the first node - start: JobID; - - // This is the execution plan for the workflow - plan: JobNode[]; -}; - -/* - -We can fetch workflow by id -The workflow is a json object -Jobs and probably state objects may be lazily resolved - -A workflow starts at the start and executes jobs until there are no more upstream jobs -*/ diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index d2a06ae86..8c1712864 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,115 +1,146 @@ import test from 'ava'; import axios from 'axios'; -import createLightningServer from '../../src/mock/lightning'; -import { wait } from '../util'; +import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; -const baseUrl = 'http://localhost:8888'; +const baseUrl = `http://localhost:8888${API_PREFIX}`; -let server = createLightningServer({ port: 8888 }); +let server; + +test.beforeEach(() => { + server = createLightningServer({ port: 8888 }); +}); + +test.afterEach(() => { + server.destroy(); +}); const get = (path: string) => axios.get(`${baseUrl}/${path}`); const post = (path: string, data: any) => axios.post(`${baseUrl}/${path}`, data); -test.serial('GET /job - return 404 if no job', async (t) => { - try { - await get('job/x'); - } catch (e) { - t.is(e.response.status, 404); - } -}); - -test.serial('GET /job - return a job', async (t) => { - const { status, data: job } = await get('job/job-1'); +test.serial('GET /credential - return a credential', async (t) => { + const { status, data: job } = await get('credential/a'); t.is(status, 200); - t.is(job.id, 'job-1'); - t.truthy(job.expression); + t.is(job.user, 'bobby'); + t.is(job.password, 'password1'); }); -test.serial('GET /job - return a new mock job', async (t) => { - server.addJob({ id: 'jam' }); - const { status, data: job } = await get('job/jam'); +test.serial('GET /credential - return a new mock credential', async (t) => { + server.addCredential('b', { user: 'johnny', password: 'cash' }); + const { status, data: job } = await get('credential/b'); t.is(status, 200); - t.is(job.id, 'jam'); + t.is(job.user, 'johnny'); + t.is(job.password, 'cash'); }); -test.serial('GET /workflow - return 404 if no workflow', async (t) => { - try { - await get('workflow/x'); - } catch (e) { - t.is(e.response.status, 404); +test.serial( + 'GET /credential - return 404 if no credential found', + async (t) => { + try { + await get('credential/c'); + } catch (e) { + t.is(e.response.status, 404); + } } -}); +); -test.serial('GET /workflow - return a workflow', async (t) => { - const { status, data: workflow } = await get('workflow/workflow-1'); - t.is(status, 200); +test.serial( + 'POST /attempts/next - return 204 for an empty queue', + async (t) => { + t.is(server.getQueueLength(), 0); + const { status, data } = await post('attempts/next', { id: 'x' }); + t.is(status, 204); - t.is(workflow.id, 'workflow-1'); + t.falsy(data); + } +); + +test.serial('POST /attempts/next - return 400 if no id provided', async (t) => { + try { + await post('attempts/next', {}); + } catch (e) { + t.is(e.response.status, 400); + } }); -test.serial('GET /workflow - return a new mock workflow', async (t) => { - server.addWorkflow({ id: 'jam' }); - const { status, data: workflow } = await get('workflow/jam'); +test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { + server.addToQueue('attempt-1'); + t.is(server.getQueueLength(), 1); + const { status, data } = await post('attempts/next', { id: 'x' }); t.is(status, 200); - t.is(workflow.id, 'jam'); -}); + t.truthy(data); + t.true(Array.isArray(data)); + t.is(data.length, 1); -test.serial('GET /queue - return 204 for an empty queue', async (t) => { - const { status, data } = await get('queue'); - t.is(status, 204); + // not interested in testing much against the attempt structure at this stage + const [attempt] = data; + t.is(attempt.id, 'attempt-1'); + t.true(Array.isArray(attempt.plan)); - t.falsy(data); + t.is(server.getQueueLength(), 0); }); -test.serial('GET /queue - return 200 with a workflow id', async (t) => { - server.addToQueue('workflow-1'); - const { status, data } = await get('queue'); +test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { + server.addToQueue('attempt-1'); + server.addToQueue('attempt-1'); + server.addToQueue('attempt-1'); + t.is(server.getQueueLength(), 3); + const { status, data } = await post('attempts/next?count=2', { id: 'x' }); t.is(status, 200); t.truthy(data); - t.is(data.workflowId, 'workflow-1'); + t.true(Array.isArray(data)); + t.is(data.length, 2); + + t.is(server.getQueueLength(), 1); }); -test.serial('GET /queue - clear the queue after a request', async (t) => { - server.addToQueue('workflow-1'); - const req1 = await get('queue'); - t.is(req1.status, 200); +test.serial( + 'POST /attempts/next - clear the queue after a request', + async (t) => { + server.addToQueue('attempt-1'); + const req1 = await post('attempts/next', { id: 'x' }); + t.is(req1.status, 200); - t.is(req1.data.workflowId, 'workflow-1'); + t.is(req1.data.length, 1); - const req2 = await get('queue'); - t.is(req2.status, 204); - t.falsy(req2.data); -}); + const req2 = await post('attempts/next', { id: 'x' }); + t.is(req2.status, 204); + t.falsy(req2.data); + } +); -test.serial('POST /notify - should return 202', async (t) => { - const { status } = await post('notify', {}); - t.is(status, 202); +test.serial('POST /attempts/notify - should return 200', async (t) => { + const { status } = await post('attempts/notify/a', {}); + t.is(status, 200); }); -test.serial('POST /notify - should echo to event emitter', async (t) => { - let evt; - let didCall = false; - //server.once('notify', (e) => (evt = e)); - server.once('notify', (e) => { - didCall = true; - evt = e; - }); - - const { status } = await post('notify', { - event: 'job-start', - workflowId: 'workflow-1', - }); - t.is(status, 202); - t.true(didCall); - // await wait(() => evt); - - // t.truthy(evt); - // t.is(evt.workflowId, 'job-start'); - // t.is(evt.workflowId, 'workflow-1'); -}); +test.serial( + 'POST /attempts/notify - should echo to event emitter', + async (t) => { + let evt; + let didCall = false; + + server.once('notify', (e) => { + didCall = true; + evt = e; + }); + + const { status } = await post('attempts/notify/a', { + event: 'job-start', + count: 101, + }); + t.is(status, 200); + t.true(didCall); + // await wait(() => evt); + + t.truthy(evt); + t.is(evt.id, 'a'); + t.is(evt.name, 'job-start'); + t.is(evt.count, 101); + } +); From 44cf555ba6c5d5d0ba17361efe56f9c21d4066b9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Mar 2023 15:22:14 +0000 Subject: [PATCH 006/232] rtm-server: Update mock implementation --- packages/rtm-server/src/api.ts | 2 + packages/rtm-server/src/mock/resolvers.ts | 26 +++ .../rtm-server/src/mock/runtime-manager.ts | 199 +++++++++++------- packages/rtm-server/src/server.ts | 24 ++- packages/rtm-server/src/types.d.ts | 38 +++- packages/rtm-server/test/integration.test.ts | 4 +- .../test/mock/runtime-manager.test.ts | 146 ++++++++----- 7 files changed, 288 insertions(+), 151 deletions(-) create mode 100644 packages/rtm-server/src/mock/resolvers.ts diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts index 021a5694d..514392a01 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api.ts @@ -5,6 +5,8 @@ import healthcheck from './middleware/healthcheck'; // So what is the API of this server? // It's mostly a pull model, apart from I think the healthcheck +// So it doesn't need much + const createAPI = () => { const router = new Router(); diff --git a/packages/rtm-server/src/mock/resolvers.ts b/packages/rtm-server/src/mock/resolvers.ts new file mode 100644 index 000000000..f328e0876 --- /dev/null +++ b/packages/rtm-server/src/mock/resolvers.ts @@ -0,0 +1,26 @@ +import type { State, Credential } from '../types'; +import { LazyResolvers } from './runtime-manager'; + +const mockResolveCredential = (_credId: string) => + new Promise((resolve) => + resolve({ + user: 'user', + password: 'pass', + }) + ); + +const mockResolveState = (_stateId: string) => + new Promise((resolve) => + resolve({ + data: {}, + }) + ); + +const mockResolveExpressions = (_stateId: string) => + new Promise((resolve) => resolve('{ data: { answer: 42 } }')); + +export default { + credentials: mockResolveCredential, + state: mockResolveState, + expressions: mockResolveExpressions, +} as LazyResolvers; diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index e5eeb2ed7..2ac02b26d 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -1,49 +1,74 @@ import { EventEmitter } from 'node:events'; -import type { State, Job, Workflow } from '../types'; +import type { State, Credential, ExecutionPlan, JobPlan } from '../types'; +import mockResolvers from './resolvers'; // A mock runtime manager -// +// Runs ExecutionPlans(XPlans) in worker threads +// May need to lazy-load resources +// The mock RTM will return expression JSON as state + +type Resolver = (id: string) => Promise; + +// A list of helper functions which basically resolve ids into JSON +// to lazy load assets +export type LazyResolvers = { + credentials: Resolver; + state: Resolver; + expressions: Resolver; +}; -// TODO do we mean job, or workflow? -// I think we have both? export type RTMEvent = | 'job-start' - | 'job-end' + | 'job-complete' | 'job-log' | 'job-error' | 'workflow-start' - | 'workflow-end'; - -type FetchWorkflowFn = (workflowId: string) => Promise; - -type FetchJobFn = (jobId: string) => Promise; + | 'workflow-complete' + | 'workflow-error'; // ? + +export type JobStartEvent = { + id: string; // job id + runId: string; // run id. Not sure we need this. +}; + +export type JobCompleteEvent = { + id: string; // job id + runId: string; // run id. Not sure we need this. + state: State; // do we really want to publish the intermediate events? Could be important, but also could be sensitive + // I suppose at this level yes, we should publish it +}; + +export type WorkflowStartEvent = { + id: string; // workflow id +}; + +export type WorkflowCompleteEvent = { + id: string; // workflow id + state?: object; + error?: any; +}; + +let jobId = 0; +const getNewJobId = () => `${++jobId}`; + +let autoServerId = 0; + +// Before we execute each job (expression), we have to build a state object +// This means squashing together the input state and the credential +// The crediential of course is the hard bit +const assembleState = () => {}; + +// Pre-process a plan +// Validate it +// Assign ids to jobs if they don't exist +const preprocessPlan = (plan: ExecutionPlan) => plan; -const mockFetchWorkflow = (workflowId: string) => - new Promise((resolve) => resolve({ job: 'job1', next: [] })); - -const mockFetchJob = (jobId: string) => - new Promise((resolve) => - resolve({ - expression: 'export default [s => s];', - state: {}, - }) - ); - -let id = 0; -const getNewJobId = () => ++id; - -// The mock will need some kind of helper function to get from a queue -// This could call out to an endpoint or use something memory -// actually i don't think that's right function createMock( - fetchWorkflow = mockFetchWorkflow, - fetchJob = mockFetchJob + serverId = autoServerId, + resolvers: LazyResolvers = mockResolvers ) { - const mockResults: Record = {}; - - // This at the moment is an aspirational API - its what I think we want - + const activeWorkflows = {} as Record; const bus = new EventEmitter(); const dispatch = (type: RTMEvent, args?: any) => { @@ -54,69 +79,81 @@ function createMock( }; const on = (event: RTMEvent, fn: (evt: any) => void) => { - bus.addListener(event, fn); + bus.on(event, fn); + }; + const once = (event: RTMEvent, fn: (evt: any) => void) => { + bus.once(event, fn); }; - // who handles workflows? - // a) we are given all the data about a workflow at once and we just chew through it - // (may not be possible?) - // b) we call out to get the inputs to each workflow when we start - // But who do we call out to? - // Arch doc says lightning, but I need to decouple this here - // Right now this is a string of job ids - // A workflow isn't just an array btw, it's a graph - // That does actually need thinking through - // There's a jobID, which is what Lightning calls the job - // And there's like an executionID or a runId, which is what the RTM calls the instance run - // These ought to be UUIDs so they're unique across RTMs - const startJob = (jobId: string) => { + const executeJob = async (job: JobPlan) => { + // TODO maybe lazy load the job from an id + const { id, expression, credential } = job; + + if (typeof credential === 'string') { + // Fetch the credntial but do nothing with it + // Maybe later we use it to assemble state + await resolvers.credentials(credential); + } + + // Does a job reallly need its own id...? Maybe. const runId = getNewJobId(); // Get the job details from lightning - fetchJob(jobId).then(() => { - // start instantly and emit as it goes - dispatch('job-start', { jobId, runId }); - - // TODO random timeout - // What is a job log? Anything emitted by the RTM I guess? - // Namespaced to compile, r/t, job etc. - // It's the json output of the logger - dispatch('job-log', { jobId, runId }); - - // TODO random timeout - const finalState = mockResults.hasOwnProperty(jobId) - ? mockResults[jobId] - : {}; - dispatch('job-end', { jobId, runId, state: finalState }); - }); - - return id; + // start instantly and emit as it goes + dispatch('job-start', { id, runId }); + + // TODO random timeout + // What is a job log? Anything emitted by the RTM I guess? + // Namespaced to compile, r/t, job etc. + // It's the json output of the logger + dispatch('job-log', { id, runId }); + + let state = {}; + // Try and parse the expression as JSON, in which case we use it as the final state + try { + state = JSON.parse(expression); + } catch (e) { + // Do nothing, it's fine + } + + dispatch('job-complete', { id, runId, state }); + + return state; }; - const startWorkflow = (workflowId: string) => { - // console.log('start-workflow', workflowId); - // Get the execution plan from lightning - dispatch('workflow-start', { workflowId }); - - dispatch('workflow-end', { workflowId }); + // Start executing an ExecutionPlan + // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity + const execute = (xplan: ExecutionPlan) => { + const { id, plan } = preprocessPlan(xplan); + activeWorkflows[id] = true; + setTimeout(async () => { + dispatch('workflow-start', { id }); + setTimeout(async () => { + // Run the first job + // TODO this need sto loop over all jobs + const state = await executeJob(plan[0]); + + setTimeout(() => { + delete activeWorkflows[id]; + dispatch('workflow-complete', { id, state }); + }, 1); + }, 1); + }, 1); }; // return a list of jobs in progress - const getStatus = () => {}; - - const _setJobResult = (jobId: string, result: any) => { - mockResults[jobId] = result; + const getStatus = () => { + return { + active: Object.keys(activeWorkflows).length, + }; }; return { + id: serverId || ++autoServerId, on, - // TODO runWorkflow? executeWorkflow? - startWorkflow, - startJob, // this is more of an internal API + once, + execute, getStatus, - - // mock APIs - _setJobResult, }; } diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 0124a9156..d71e6c2b4 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -13,11 +13,13 @@ import createAPI from './api'; import createMockRTM from './mock/runtime-manager'; -// This loop will call out to ask for work, with a backof -const workerLoop = async (url: string, rtm: any) => { +// This loop will call out to ask for work, with a backoff +const workBackoffLoop = async (lightningUrl: string, rtm: any) => { let timeout = 100; // TODO strange stuff happens if this has a low value - console.log(`${url}/queue`); - const result = await axios.get(`${url}/queue`); + + const result = await axios.post(`${lightningUrl}/api/1/attempts`, { + id: rtm.id, + }); if (result.data) { console.log(result.data); rtm.startWorkflow(result.data.workflowId); @@ -25,19 +27,19 @@ const workerLoop = async (url: string, rtm: any) => { timeout = timeout * 2; } setTimeout(() => { - workerLoop(url, rtm); + workBackoffLoop(lightningUrl, rtm); }, timeout); }; type ServerOptions = { - backoff: number; - maxWorkflows: number; - port: number; - lightning: string; // url to lightning instance + backoff?: number; + maxWorkflows?: number; + port?: number; + lightning?: string; // url to lightning instance rtm?: any; }; -function createServer(options = {}) { +function createServer(options: ServerOptions = {}) { const app = new Koa(); const rtm = options.rtm || new createMockRTM(); @@ -49,7 +51,7 @@ function createServer(options = {}) { app.listen(options.port || 1234); if (options.lightning) { - workerLoop(options.lightning, rtm); + workBackoffLoop(options.lightning, rtm); } // TMP doing this for tests but maybe its better done externally diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index 441e5313b..d7ffc3628 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -1,10 +1,10 @@ -export type Credential = object; +export type Credential = Record; export type State = { data: { [key: string]: any; }; - configuration: { + configuration?: { [key: string]: any; }; @@ -17,9 +17,10 @@ export type State = { export type LightningAttempt = { id: string; input: Omit; // initial state - plan: LightningJob; + plan: LightningJob[]; // these will probably be included by lightning but we don't care here + projectId: string; status: string; worker: string; }; @@ -37,3 +38,34 @@ export type LightningJob = { default: LightningJob; }; }; + +type RuntimeExecutionPlanID = string; + +// TODO this type should later be imported from the runtime +type JobPlan = { + id: string; + adaptor: string; + expression: string; // the code we actually want to execute. Could be lazy loaded + credential: string | object; // credential can be inline or lazy loaded + state?: Omit; // initial state + + upstream: + | JobPlanID // shorthand for { default } + | { + success: JobPlanID; + error: JobPlanID; + default: JobPlanID; + }; +}; + +// A runtime manager execution plan +export type ExecutionPlan = { + id: string; // UUID for this plan + + // should we save the initial and resulting status? + // Should we have a status here, is this a living thing? + + plan: JobPlan[]; // TODO this type should later be imported from the runtime +}; + +export type Xplan = ExecutionPlan; diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 0bd3b7dd6..65e285433 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -20,10 +20,10 @@ test.before(() => { }); // TODO get this working -test.serial('should pick up a workflow in the queue', async (t) => { +test.serial('should pick up an attempt in the queue', async (t) => { let found: false | string = false; - rtm.on('workflow-start', (e) => { + rtm.on('attempt-start', (e) => { // console.log(e); found = e.workflowId; }); diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index eb2f636b2..5722d9333 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -1,89 +1,127 @@ import test from 'ava'; -import create from '../../src/mock/runtime-manager'; - +import create, { + JobCompleteEvent, + JobStartEvent, + WorkflowCompleteEvent, + WorkflowStartEvent, +} from '../../src/mock/runtime-manager'; +import { ExecutionPlan } from '../../src/types'; import { wait } from '../util'; -test('It should create a mock runtime manager', (t) => { - const rtm = create(); +const sampleWorkflow = { + id: 'w1', + plan: [ + { + id: 'j1', + adaptor: 'common@1.0.0', + expression: '{ "x": 10 }', + }, + ], +} as ExecutionPlan; + +const clone = (obj) => JSON.parse(JSON.stringify(obj)); + +const waitForEvent = (rtm, eventName) => + new Promise((resolve) => { + rtm.once(eventName, (e) => { + resolve(e); + }); + }); + +test('mock runtime manager should have an id', (t) => { + const rtm = create(22); const keys = Object.keys(rtm); + t.assert(rtm.id == 22); + + // No need to test the full API, just make sure it smells right t.assert(keys.includes('on')); - t.assert(keys.includes('startWorkflow')); - t.assert(keys.includes('startJob')); - t.assert(keys.includes('getStatus')); + t.assert(keys.includes('execute')); }); -test('it should dispatch job-start events', async (t) => { - const rtm = create(); - - let evt; +test('getStatus() should should have no active workflows', (t) => { + const rtm = create(22); + const { active } = rtm.getStatus(); - rtm.on('job-start', (e) => { - evt = e; - }); + t.is(active, 0); +}); - rtm.startJob('a'); +test('Dispatch start events for a new workflow', async (t) => { + const rtm = create(); - const didFire = await wait(() => evt); - t.true(didFire); - t.is(evt.jobId, 'a'); - t.truthy(evt.runId); + rtm.execute(sampleWorkflow); + const evt = await waitForEvent(rtm, 'workflow-start'); + t.truthy(evt); + t.is(evt.id, 'w1'); }); -test('it should dispatch job-log events', async (t) => { +test('getStatus should report one active workflow', async (t) => { const rtm = create(); + rtm.execute(sampleWorkflow); - let evt; + const { active } = rtm.getStatus(); - rtm.on('job-log', (e) => { - evt = e; - }); + t.is(active, 1); +}); - rtm.startJob('a'); +test('Dispatch complete events when a workflow completes', async (t) => { + const rtm = create(); - const didFire = await wait(() => evt); + rtm.execute(sampleWorkflow); + const evt = await waitForEvent( + rtm, + 'workflow-complete' + ); - t.true(didFire); - t.is(evt.jobId, 'a'); - t.truthy(evt.runId); + t.truthy(evt); + t.is(evt.id, 'w1'); + t.truthy(evt.state); }); -test('it should dispatch job-end events', async (t) => { +test('Dispatch start events for a job', async (t) => { const rtm = create(); - let evt; - - rtm.on('job-end', (e) => { - evt = e; - }); - - rtm.startJob('a'); + rtm.execute(sampleWorkflow); + const evt = await waitForEvent(rtm, 'job-start'); + t.truthy(evt); + t.is(evt.id, 'j1'); + t.truthy(evt.runId); +}); - const didFire = await wait(() => evt); +test('Dispatch complete events for a job', async (t) => { + const rtm = create(); - t.true(didFire); - t.is(evt.jobId, 'a'); + rtm.execute(sampleWorkflow); + const evt = await waitForEvent(rtm, 'job-complete'); + t.truthy(evt); + t.is(evt.id, 'j1'); t.truthy(evt.runId); t.truthy(evt.state); }); -test('it should mock job state', async (t) => { +test('mock should evaluate expressions as JSON', async (t) => { const rtm = create(); - const result = 42; - - rtm._setJobResult('a', result); - let evt; + rtm.execute(sampleWorkflow); + const evt = await waitForEvent( + rtm, + 'workflow-complete' + ); + t.deepEqual(evt.state, { x: 10 }); +}); - rtm.on('job-end', (e) => { - evt = e; - }); +test('resolve credential before job-start if credential is a string', async (t) => { + const wf = clone(sampleWorkflow); + wf.plan[0].credential = 'x'; - rtm.startJob('a'); + let didCallCredentials; + const credentials = async (_id) => { + didCallCredentials = true; + return {}; + }; - const didFire = await wait(() => evt); + const rtm = create(1, { credentials }); + rtm.execute(wf); - t.true(didFire); - t.is(evt.jobId, 'a'); - t.truthy(evt.runId); - t.is(evt.state, result); + await waitForEvent(rtm, 'job-start'); + t.true(didCallCredentials); }); From 1cd3c6c0d93ca6c71e3c4e211e7aec74f65c4401 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Mar 2023 15:55:17 +0000 Subject: [PATCH 007/232] rtm: refactor lightning mock for a nicer seperation of api and logic --- packages/rtm-server/ava | 0 packages/rtm-server/axios | 0 packages/rtm-server/src/mock/lightning.ts | 154 ------------------ packages/rtm-server/src/mock/lightning/api.ts | 46 ++++++ .../rtm-server/src/mock/lightning/index.ts | 4 + .../src/mock/lightning/middleware.ts | 80 +++++++++ .../rtm-server/src/mock/lightning/server.ts | 75 +++++++++ 7 files changed, 205 insertions(+), 154 deletions(-) create mode 100644 packages/rtm-server/ava create mode 100644 packages/rtm-server/axios delete mode 100644 packages/rtm-server/src/mock/lightning.ts create mode 100644 packages/rtm-server/src/mock/lightning/api.ts create mode 100644 packages/rtm-server/src/mock/lightning/index.ts create mode 100644 packages/rtm-server/src/mock/lightning/middleware.ts create mode 100644 packages/rtm-server/src/mock/lightning/server.ts diff --git a/packages/rtm-server/ava b/packages/rtm-server/ava new file mode 100644 index 000000000..e69de29bb diff --git a/packages/rtm-server/axios b/packages/rtm-server/axios new file mode 100644 index 000000000..e69de29bb diff --git a/packages/rtm-server/src/mock/lightning.ts b/packages/rtm-server/src/mock/lightning.ts deleted file mode 100644 index e6e77d72c..000000000 --- a/packages/rtm-server/src/mock/lightning.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { EventEmitter } from 'node:events'; -import Koa from 'koa'; -import bodyParser from 'koa-bodyparser'; -import Router from '@koa/router'; - -import * as data from './data'; -import { LightningAttempt, State } from '../types'; -import { RTMEvent } from './runtime-manager'; - -const unimplemented = (ctx) => { - ctx.statusCode = 501; -}; - -type NotifyEvent = { - event: RTMEvent; - workflow: string; // workflow id - [key: string]: any; -}; - -export const API_PREFIX = `/api/v1`; - -// a mock lightning server -const createLightningServer = (options = {}) => { - // App state - const credentials = data.credentials(); - const attempts = data.attempts(); - const queue: string[] = []; - const results = {}; - - // Server setup - const app = new Koa(); - const router = new Router(); - const events = new EventEmitter(); - app.use(bodyParser()); - - // Mock API endpoints - - // POST attempts/next: - // 200 - return an array of pending attempts - // 204 - queue empty (no body) - router.post(`${API_PREFIX}/attempts/next`, (ctx) => { - const { body } = ctx.request; - if (!body || !body.id) { - ctx.status = 400; - return; - } - - let count = ctx.request.query.count || 1; - const payload = []; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - payload.push(queue.shift()); - count -= 1; - } - - if (payload.length > 0) { - ctx.body = JSON.stringify(payload); - ctx.status = 200; - } else { - ctx.body = undefined; - ctx.status = 204; - } - }); - - // GET credential/:id - // 200 - return a credential object - // 404 - credential not found - router.get(`${API_PREFIX}/credential/:id`, (ctx) => { - const cred = credentials[ctx.params.id]; - if (cred) { - ctx.body = JSON.stringify(cred); - ctx.status = 200; - } else { - ctx.status = 404; - } - }); - - // Notify of some job update - // proxy to event emitter - // { event: 'event-name', ...data } - router.post(`${API_PREFIX}/attempts/notify/:id`, (ctx) => { - const { event: name, ...payload } = ctx.request.body; - - const event = { - id: ctx.params.id, - name, - ...payload, // spread payload? - }; - - events.emit('notify', event); - - ctx.status = 200; - }); - - // Notify an attempt has finished - // Could be error or success state - // Error or state in payload - // { data } | { error } - router.post(`${API_PREFIX}/attempts/complete/:id`, (ctx) => { - const finalState = ctx.data as State; - - results[ctx.params.id] = finalState; - - ctx.status = 200; - }); - - // Unimplemented API endpoints - - router.get(`${API_PREFIX}/attempts/:id`, unimplemented); - router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 - router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid - - router.get(`${API_PREFIX}/credential/:id`, unimplemented); - router.get(`${API_PREFIX}/attempts/active`, unimplemented); - - router.get(`${API_PREFIX}/workflows`, unimplemented); - router.get(`${API_PREFIX}/workflows/:id`, unimplemented); - - app.use(router.routes()); - - // Dev APIs for unit testing - app.addCredential = (id: string, cred: Credential) => { - credentials[id] = cred; - }; - app.addAttempt = (attempt: LightningAttempt) => { - attempts[attempt.id] = attempt; - }; - app.addToQueue = (attemptId: string) => { - if (attempts[attemptId]) { - queue.push(attempts[attemptId]); - return true; - } - return false; - }; - app.getQueueLength = () => queue.length; - app.on = (event: 'notify', fn: (evt: any) => void) => { - events.addListener(event, fn); - }; - app.once = (event: 'notify', fn: (evt: any) => void) => { - events.once(event, fn); - }; - - const server = app.listen(options.port || 8888); - - app.destroy = () => { - server.close(); - }; - - return app; -}; - -export default createLightningServer; diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts new file mode 100644 index 000000000..de64da542 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -0,0 +1,46 @@ +import type Router from '@koa/router'; +import { + unimplemented, + createFetchNextJob, + createGetCredential, + createNotify, + createComplete, +} from './middleware'; +import type { ServerState } from './server'; + +export const API_PREFIX = `/api/v1`; + +export default (router: Router, state: ServerState) => { + // POST attempts/next: + // 200 - return an array of pending attempts + // 204 - queue empty (no body) + router.post(`${API_PREFIX}/attempts/next`, createFetchNextJob(state)); + + // GET credential/:id + // 200 - return a credential object + // 404 - credential not found + router.get(`${API_PREFIX}/credential/:id`, createGetCredential(state)); + + // Notify of some job update + // proxy to event emitter + // { event: 'event-name', ...data } + router.post(`${API_PREFIX}/attempts/notify/:id`, createNotify(state)); + + // Notify an attempt has finished + // Could be error or success state + // Error or state in payload + // { data } | { error } + router.post(`${API_PREFIX}/attempts/complete/:id`, createComplete(state)); + + router.get(`${API_PREFIX}/attempts/:id`, unimplemented); + router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 + router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid + + router.get(`${API_PREFIX}/credential/:id`, unimplemented); + router.get(`${API_PREFIX}/attempts/active`, unimplemented); + + router.get(`${API_PREFIX}/workflows`, unimplemented); + router.get(`${API_PREFIX}/workflows/:id`, unimplemented); + + return router; +}; diff --git a/packages/rtm-server/src/mock/lightning/index.ts b/packages/rtm-server/src/mock/lightning/index.ts new file mode 100644 index 000000000..dc631ee27 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/index.ts @@ -0,0 +1,4 @@ +import createLightningServer from './server'; +export default createLightningServer; + +export { API_PREFIX } from './api'; diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts new file mode 100644 index 000000000..afd2b8669 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -0,0 +1,80 @@ +import Koa from 'koa'; +import type { ServerState } from './server'; +import { State } from '../../types'; + +export const unimplemented = (ctx: Koa.Context) => { + ctx.status = 501; +}; + +export const createFetchNextJob = + (state: ServerState) => (ctx: Koa.Context) => { + const { queue } = state; + + const { body } = ctx.request; + if (!body || !body.id) { + ctx.status = 400; + return; + } + + const countRaw = ctx.request.query.count as unknown; + let count = 1; + if (countRaw) { + if (!isNaN(countRaw)) { + count = countRaw as number; + } else { + console.error('Failed to parse parameter countRaw'); + } + } + const payload = []; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + payload.push(queue.shift()); + count -= 1; + } + + if (payload.length > 0) { + ctx.body = JSON.stringify(payload); + ctx.status = 200; + } else { + ctx.body = undefined; + ctx.status = 204; + } + }; + +export const createGetCredential = + (state: ServerState) => (ctx: Koa.Context) => { + const { credentials } = state; + const cred = credentials[ctx.params.id]; + if (cred) { + ctx.body = JSON.stringify(cred); + ctx.status = 200; + } else { + ctx.status = 404; + } + }; + +export const createNotify = (state: ServerState) => (ctx: Koa.Context) => { + const { events } = state; + const { event: name, ...payload } = ctx.request.body; + + const event = { + id: ctx.params.id, + name, + ...payload, // spread payload? + }; + + events.emit('notify', event); + + ctx.status = 200; +}; + +export const createComplete = (state: ServerState) => (ctx: Koa.Context) => { + const { results } = state; + const finalState = ctx.data as State; + + results[ctx.params.id] = finalState; + + ctx.status = 200; +}; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts new file mode 100644 index 000000000..0d5b03a26 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -0,0 +1,75 @@ +import { EventEmitter } from 'node:events'; +import Koa from 'koa'; +import bodyParser from 'koa-bodyparser'; +import Router from '@koa/router'; + +import createAPI from './api'; +import * as data from '../data'; +import { LightningAttempt } from '../../types'; +import { RTMEvent } from '../runtime-manager'; + +type NotifyEvent = { + event: RTMEvent; + workflow: string; // workflow id + [key: string]: any; +}; + +export type ServerState = { + credentials: Record; + attempts: Record; + queue: LightningAttempt[]; + results: Record; + events: EventEmitter; +}; + +// a mock lightning server +const createLightningServer = (options = {}) => { + // App state + const state = { + credentials: data.credentials(), + attempts: data.attempts(), + queue: [] as LightningAttempt[], + results: {}, + events: new EventEmitter(), + } as ServerState; + + // Server setup + const app = new Koa(); + const api = createAPI(new Router(), state); + app.use(bodyParser()); + + // Mock API endpoints + app.use(api.routes()); + + // Dev APIs for unit testing + app.addCredential = (id: string, cred: Credential) => { + state.credentials[id] = cred; + }; + app.addAttempt = (attempt: LightningAttempt) => { + state.attempts[attempt.id] = attempt; + }; + app.addToQueue = (attemptId: string) => { + if (state.attempts[attemptId]) { + state.queue.push(state.attempts[attemptId]); + return true; + } + return false; + }; + app.getQueueLength = () => state.queue.length; + app.on = (event: 'notify', fn: (evt: any) => void) => { + state.events.addListener(event, fn); + }; + app.once = (event: 'notify', fn: (evt: any) => void) => { + state.events.once(event, fn); + }; + + const server = app.listen(options.port || 8888); + + app.destroy = () => { + server.close(); + }; + + return app; +}; + +export default createLightningServer; From 68e47ccd26757ced6ca5f9a1888d8be7c93e7f93 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Mar 2023 15:55:51 +0000 Subject: [PATCH 008/232] rtm: tweak api layout --- packages/rtm-server/src/mock/lightning/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index de64da542..55754625a 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -35,9 +35,9 @@ export default (router: Router, state: ServerState) => { router.get(`${API_PREFIX}/attempts/:id`, unimplemented); router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid + router.get(`${API_PREFIX}/attempts/active`, unimplemented); router.get(`${API_PREFIX}/credential/:id`, unimplemented); - router.get(`${API_PREFIX}/attempts/active`, unimplemented); router.get(`${API_PREFIX}/workflows`, unimplemented); router.get(`${API_PREFIX}/workflows/:id`, unimplemented); From d643fbaca09f2d3211bbb0d111064cc3f585ce17 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 24 Mar 2023 18:17:15 +0000 Subject: [PATCH 009/232] rtm: hook up integration tests for glorious victory --- packages/rtm-server/src/mock/lightning/api.ts | 2 +- .../src/mock/lightning/middleware.ts | 7 +- .../rtm-server/src/mock/lightning/server.ts | 14 ++-- packages/rtm-server/src/server.ts | 26 +++++++- packages/rtm-server/test/integration.test.ts | 66 ++++++++++++++----- .../rtm-server/test/mock/lightning.test.ts | 55 ++++++++++++++++ .../test/mock/runtime-manager.test.ts | 11 +--- packages/rtm-server/test/server.test.ts | 3 + packages/rtm-server/test/util.ts | 16 ++++- 9 files changed, 159 insertions(+), 41 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 55754625a..ece1f72c0 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -8,7 +8,7 @@ import { } from './middleware'; import type { ServerState } from './server'; -export const API_PREFIX = `/api/v1`; +export const API_PREFIX = `/api/1`; export default (router: Router, state: ServerState) => { // POST attempts/next: diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index afd2b8669..c86322f09 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -71,10 +71,11 @@ export const createNotify = (state: ServerState) => (ctx: Koa.Context) => { }; export const createComplete = (state: ServerState) => (ctx: Koa.Context) => { - const { results } = state; - const finalState = ctx.data as State; - + const { results, events } = state; + const finalState = ctx.request.body as State; results[ctx.params.id] = finalState; + events.emit('complete', { id: ctx.params.id, state: finalState }); + ctx.status = 200; }; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 0d5b03a26..1444546d4 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -48,14 +48,20 @@ const createLightningServer = (options = {}) => { app.addAttempt = (attempt: LightningAttempt) => { state.attempts[attempt.id] = attempt; }; - app.addToQueue = (attemptId: string) => { - if (state.attempts[attemptId]) { - state.queue.push(state.attempts[attemptId]); + app.addToQueue = (attempt: string | LightningAttempt) => { + if (typeof attempt == 'string') { + if (state.attempts[attempt]) { + state.queue.push(state.attempts[attempt]); + return true; + } + throw new Error(`attempt ${attempt} not found`); + } else if (attempt) { + state.queue.push(attempt); return true; } - return false; }; app.getQueueLength = () => state.queue.length; + app.getResult = (attemptId: string) => state.results[attemptId]; app.on = (event: 'notify', fn: (evt: any) => void) => { state.events.addListener(event, fn); }; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index d71e6c2b4..1a6b497d9 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -17,12 +17,14 @@ import createMockRTM from './mock/runtime-manager'; const workBackoffLoop = async (lightningUrl: string, rtm: any) => { let timeout = 100; // TODO strange stuff happens if this has a low value - const result = await axios.post(`${lightningUrl}/api/1/attempts`, { + const result = await axios.post(`${lightningUrl}/api/1/attempts/next`, { id: rtm.id, }); + if (result.data) { - console.log(result.data); - rtm.startWorkflow(result.data.workflowId); + // TODO handle multiple attempts + const [attempt] = result.data; + rtm.execute(attempt); } else { timeout = timeout * 2; } @@ -31,6 +33,19 @@ const workBackoffLoop = async (lightningUrl: string, rtm: any) => { }, timeout); }; +const postResult = async ( + lightningUrl: string, + attemptId: string, + state: any +) => { + const result = await axios.post( + `${lightningUrl}/api/1/attempts/complete/${attemptId}`, + state || {} + ); + // TODO what if result is not 200? + // Backoff and try again? +}; + type ServerOptions = { backoff?: number; maxWorkflows?: number; @@ -52,10 +67,15 @@ function createServer(options: ServerOptions = {}) { if (options.lightning) { workBackoffLoop(options.lightning, rtm); + + rtm.on('workflow-complete', ({ id, state }) => { + postResult(options.lightning!, id, state); + }); } // TMP doing this for tests but maybe its better done externally app.on = (...args) => rtm.on(...args); + app.once = (...args) => rtm.once(...args); return app; diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 65e285433..732dca8cb 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -1,35 +1,67 @@ -// This will test both servers talking to each other +// these integration tests test the realy rtm server logic with a mock lightning and a mock rtm +// So maybe it's not really "integration" tests after all, but regular server tests import test from 'ava'; import axios from 'axios'; import createRTMServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; -import { wait } from './util'; +import { wait, waitForEvent } from './util'; let lng; let rtm; const urls = { - rtm: 'http://localhost:7777', - lng: 'http://localhost:8888', + rtm: 'http://localhost:4567', + lng: 'http://localhost:7654', }; test.before(() => { - lng = createLightningServer({ port: 8888 }); - rtm = createRTMServer({ port: 7777, lightning: urls.lng }); + lng = createLightningServer({ port: 7654 }); + rtm = createRTMServer({ port: 4567, lightning: urls.lng }); }); -// TODO get this working -test.serial('should pick up an attempt in the queue', async (t) => { - let found: false | string = false; - - rtm.on('attempt-start', (e) => { - // console.log(e); - found = e.workflowId; - }); +test.serial('should pick up a default attempt in the queue', async (t) => { + lng.addToQueue('attempt-1'); + const evt = await waitForEvent(rtm, 'workflow-start'); + t.truthy(evt); + t.is(evt.id, 'attempt-1'); +}); - lng.addToQueue('a'); +test.serial('should pick up a novel attempt in the queue', async (t) => { + lng.addToQueue({ id: 'x', plan: [{ expression: '{}' }] }); + const evt = await waitForEvent(rtm, 'workflow-start'); + t.truthy(evt); + t.is(evt.id, 'x'); - await wait(() => found); - t.is(found as unknown as string, 'a'); + // let the workflow finish processing + await waitForEvent(rtm, 'workflow-complete'); }); + +test.serial( + 'should publish a workflow-complete event with state', + async (t) => { + // The mock RTM will evaluate the expression as JSON and return it + lng.addToQueue({ id: 'x', plan: [{ expression: '{ "answer": 42 }' }] }); + + const evt = await waitForEvent(rtm, 'workflow-complete'); + t.truthy(evt); + t.is(evt.id, 'x'); + t.deepEqual(evt.state, { answer: 42 }); + } +); + +test.serial( + 'should post to attempts/complete with the final state', + async (t) => { + // The mock RTM will evaluate the expression as JSON and return it + lng.addToQueue({ id: 'y', plan: [{ expression: '{ "answer": 42 }' }] }); + + await waitForEvent(rtm, 'workflow-complete'); + + // The RMT server will post to attempts/complete/:id with the state, which should eventually + // be available to our little debug API here + const result = await wait(() => lng.getResult('y')); + t.truthy(result); + t.is(result.answer, 42); + } +); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 8c1712864..0f7ee0ec5 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -84,6 +84,25 @@ test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { t.is(server.getQueueLength(), 0); }); +test.serial( + 'GET /attempts/next - return 200 with a workflow with an inline item', + async (t) => { + server.addToQueue({ id: 'abc' }); + t.is(server.getQueueLength(), 1); + const { status, data } = await post('attempts/next', { id: 'x' }); + t.is(status, 200); + + t.truthy(data); + t.true(Array.isArray(data)); + t.is(data.length, 1); + + const [attempt] = data; + t.is(attempt.id, 'abc'); + + t.is(server.getQueueLength(), 0); + } +); + test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { server.addToQueue('attempt-1'); server.addToQueue('attempt-1'); @@ -144,3 +163,39 @@ test.serial( t.is(evt.count, 101); } ); + +test.serial('POST /attempts/complete - return final state', async (t) => { + const { status } = await post('attempts/complete/a', { + x: 10, + }); + t.is(status, 200); + const result = server.getResult('a'); + t.deepEqual(result, { x: 10 }); +}); + +test.serial( + 'POST /attempts/complete - should echo to event emitter', + async (t) => { + let evt; + let didCall = false; + + server.once('complete', (e) => { + didCall = true; + evt = e; + }); + + const { status } = await post('attempts/complete/a', { + data: { + answer: 42, + }, + }); + t.is(status, 200); + t.true(didCall); + + t.truthy(evt); + t.is(evt.id, 'a'); + t.deepEqual(evt.state, { data: { answer: 42 } }); + } +); + +// test lightning should get the finished state through a helper API diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index 5722d9333..cf71a5d07 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -6,7 +6,7 @@ import create, { WorkflowStartEvent, } from '../../src/mock/runtime-manager'; import { ExecutionPlan } from '../../src/types'; -import { wait } from '../util'; +import { waitForEvent, clone } from '../util'; const sampleWorkflow = { id: 'w1', @@ -19,15 +19,6 @@ const sampleWorkflow = { ], } as ExecutionPlan; -const clone = (obj) => JSON.parse(JSON.stringify(obj)); - -const waitForEvent = (rtm, eventName) => - new Promise((resolve) => { - rtm.once(eventName, (e) => { - resolve(e); - }); - }); - test('mock runtime manager should have an id', (t) => { const rtm = create(22); const keys = Object.keys(rtm); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index a7b18dbae..f1a9dbd92 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -2,6 +2,9 @@ import test from 'ava'; import axios from 'axios'; import createServer from '../src/server'; +// Unit tests against the RTM web server +// I don't think there will ever be much here because the server is mostly a pull + let server; const url = 'http://localhost:7777'; diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index aac8ae618..839ef533f 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -3,14 +3,24 @@ export const wait = (fn, maxAttempts = 100) => let count = 0; let ival = setInterval(() => { count++; - if (fn()) { + const result = fn() || true; + if (result) { clearInterval(ival); - resolve(true); + resolve(result); } if (count == maxAttempts) { clearInterval(ival); - resolve(false); + resolve(); } }, 100); }); + +export const clone = (obj) => JSON.parse(JSON.stringify(obj)); + +export const waitForEvent = (rtm, eventName) => + new Promise((resolve) => { + rtm.once(eventName, (e) => { + resolve(e); + }); + }); From 9806f57bd5a0b6f1419cf5c6e9283fe50ad7a6da Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 29 Mar 2023 15:59:10 +0100 Subject: [PATCH 010/232] rtm-server: refactoring out the core worker loop I think this is better? --- packages/rtm-server/src/server.ts | 6 +- packages/rtm-server/src/util.ts | 50 ++++++++++++++++ packages/rtm-server/src/work-loop.ts | 45 ++++++++++++++ packages/rtm-server/test/integration.test.ts | 3 +- .../test/util/try-with-backoff.test.ts | 58 +++++++++++++++++++ 5 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 packages/rtm-server/src/util.ts create mode 100644 packages/rtm-server/src/work-loop.ts create mode 100644 packages/rtm-server/test/util/try-with-backoff.test.ts diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 1a6b497d9..0f3e82b45 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -10,7 +10,7 @@ import Koa from 'koa'; import axios from 'axios'; import createAPI from './api'; - +import startWorkLoop from './work-loop'; import createMockRTM from './mock/runtime-manager'; // This loop will call out to ask for work, with a backoff @@ -66,11 +66,13 @@ function createServer(options: ServerOptions = {}) { app.listen(options.port || 1234); if (options.lightning) { - workBackoffLoop(options.lightning, rtm); + startWorkLoop(options.lightning, rtm); rtm.on('workflow-complete', ({ id, state }) => { postResult(options.lightning!, id, state); }); + + // TODO how about an 'all' so we can "route" events? } // TMP doing this for tests but maybe its better done externally diff --git a/packages/rtm-server/src/util.ts b/packages/rtm-server/src/util.ts new file mode 100644 index 000000000..dd0392ca8 --- /dev/null +++ b/packages/rtm-server/src/util.ts @@ -0,0 +1,50 @@ +// re-usable function which will try a thing repeatedly +// TODO take a timeout + +type Options = { + attempts?: number; + maxAttempts?: number; + maxBackoff?: number; + timeout?: number; +}; + +// what is the API to this? +// Function should throw if it fails +// but in the main work loop it's not reall a fail for no work +// And we should back off +// under what circumstance should this function throw? +// If it timesout +// Can the inner function force a throw? An exit early? +export const tryWithBackoff = (fn, opts: Options = {}) => { + if (!opts.timeout) { + opts.timeout = 100; // TODO errors occur if this is too low? + } + if (!opts.attempts) { + opts.attempts = 1; + } + let { timeout, attempts, maxAttempts } = opts; + timeout = timeout || 1; + attempts = attempts || 1; + + return new Promise(async (resolve, reject) => { + try { + await fn(); + resolve(); + } catch (e) { + if (!isNaN(maxAttempts as any) && attempts >= (maxAttempts as number)) { + return reject(new Error('max attempts exceeded')); + } + // failed? No problem, we'll back off and try again + // TODO update opts + // TODO is this gonna cause a crazy promise chain? + setTimeout(() => { + const nextOpts = { + maxAttempts, + attempts: attempts + 1, + timeout: timeout * 2, + }; + tryWithBackoff(fn, nextOpts).then(resolve).catch(reject); + }, timeout); + } + }); +}; diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts new file mode 100644 index 000000000..cab69c397 --- /dev/null +++ b/packages/rtm-server/src/work-loop.ts @@ -0,0 +1,45 @@ +// idk what to call this +// Does it count as part of the API? +// Is it a kind of middleware? +// Is it one of many RTM helpers? + +import axios from 'axios'; +import { tryWithBackoff } from './util'; + +// TODO how does this report errors, like if Lightning is down? +// Or auth is bad? +export default (lightningUrl: string, rtm: any) => { + const fetchWork = async () => { + // TODO what if this retuns like a 500? Server down? + const result = await axios.post(`${lightningUrl}/api/1/attempts/next`, { + id: rtm.id, + }); + if (result.data) { + // TODO handle multiple attempts + const [attempt] = result.data; + // Start the job (but don't wtit for it) + rtm.execute(attempt); + + // Return true t + return true; + } + // return false to backoff and try again + return false; + }; + + const workLoop = () => { + tryWithBackoff(fetchWork) + .then(workLoop) + .catch(() => { + // this means the backoff expired + // which right now it won't ever do + // but what's the plan? + // log and try again I guess? + workLoop(); + }); + }; + + return workLoop(); + // maybe we can return an api like + // { start, pause, on('error') } +}; diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 732dca8cb..0f9cff90b 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -1,7 +1,6 @@ -// these integration tests test the realy rtm server logic with a mock lightning and a mock rtm +// these integration tests test the real rtm server logic with a mock lightning and a mock rtm // So maybe it's not really "integration" tests after all, but regular server tests import test from 'ava'; -import axios from 'axios'; import createRTMServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; diff --git a/packages/rtm-server/test/util/try-with-backoff.test.ts b/packages/rtm-server/test/util/try-with-backoff.test.ts new file mode 100644 index 000000000..8516e1e40 --- /dev/null +++ b/packages/rtm-server/test/util/try-with-backoff.test.ts @@ -0,0 +1,58 @@ +import test from 'ava'; + +import { tryWithBackoff } from '../../src/util'; + +test('return immediately', async (t) => { + let callCount = 0; + const fn = async () => { + callCount++; + }; + + await tryWithBackoff(fn); + t.is(callCount, 1); +}); + +test('return on second try', async (t) => { + let callCount = 0; + const fn = () => { + callCount++; + if (callCount <= 1) { + throw new Error('test'); + } + }; + + await tryWithBackoff(fn); + + t.is(callCount, 2); +}); + +test.skip('return on tenth try (maximum backoff)', () => {}); + +test('throw if maximum attempts (1) reached', async (t) => { + let callCount = 0; + const fn = async () => { + callCount++; + throw new Error('test'); + }; + + await t.throwsAsync(() => tryWithBackoff(fn, { maxAttempts: 1 }), { + message: 'max attempts exceeded', + }); + t.is(callCount, 1); +}); + +test('throw if maximum attempts (5) reached', async (t) => { + let callCount = 0; + const fn = async () => { + callCount++; + throw new Error('test'); + }; + + await t.throwsAsync(() => tryWithBackoff(fn, { maxAttempts: 5 }), { + message: 'max attempts exceeded', + }); + t.is(callCount, 5); +}); + +// TODO allow to be cancelled +// TODO test increasing backoffs From fc241e018fde13d68ea5c34e468041b12c3f2076 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 25 May 2023 18:24:11 +0100 Subject: [PATCH 011/232] rtm-server: big refactor - use Lightning view of Attempt and convert it to an ExecutionPlan - start restructuring tests to be more consistent and readable - remove some unused stuff - update notes with better architectural docs - RTM has to be passed into the server now, it no longer creates its own mock --- packages/rtm-server/ava | 0 packages/rtm-server/axios | 0 packages/rtm-server/notes | 55 ++++- packages/rtm-server/package.json | 1 + packages/rtm-server/src/mock/lightning/api.ts | 7 + .../rtm-server/src/mock/runtime-manager.ts | 8 +- packages/rtm-server/src/server.ts | 71 +++--- packages/rtm-server/src/types.d.ts | 99 ++++---- .../rtm-server/src/util/convert-attempt.ts | 65 +++++ packages/rtm-server/src/util/index.ts | 4 + .../src/{util.ts => util/try-with-backoff.ts} | 4 +- packages/rtm-server/src/work-loop.ts | 23 +- packages/rtm-server/test/events.test.ts | 53 +++++ packages/rtm-server/test/integration.test.ts | 42 ++-- .../test/mock/runtime-manager.test.ts | 4 +- .../test/util/convert-attempt.test.ts | 222 ++++++++++++++++++ 16 files changed, 512 insertions(+), 146 deletions(-) delete mode 100644 packages/rtm-server/ava delete mode 100644 packages/rtm-server/axios create mode 100644 packages/rtm-server/src/util/convert-attempt.ts create mode 100644 packages/rtm-server/src/util/index.ts rename packages/rtm-server/src/{util.ts => util/try-with-backoff.ts} (94%) create mode 100644 packages/rtm-server/test/events.test.ts create mode 100644 packages/rtm-server/test/util/convert-attempt.test.ts diff --git a/packages/rtm-server/ava b/packages/rtm-server/ava deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/rtm-server/axios b/packages/rtm-server/axios deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes index 3fb2712d6..4a5d1ca78 100644 --- a/packages/rtm-server/notes +++ b/packages/rtm-server/notes @@ -1,19 +1,52 @@ -lightning must today have a model and execution plan for a workflow +The Runtime Manager Server provides a HTTP interface between Lightning and a Runtime Manager. -It needs to know where and when to start (triggers will still be handled by Lightning), and on completion, what to do next +It is a fairly thin layer between the two systems. -I'm gonna have to re-implement that in js] +It should ideally have zero persistence. -The more we speak to lightning, a) the tighter the coupling and b) the greater impact of downtime +## General Architecture -right now it looks like we have to: +Lightning provides an endpoint to fetch any pending jobs from a queue. -- Ask lightning for a workflow id -- Now ask lightning for an execution plan -- On each job, ask lightning for the expression an state +The RTM server polls this endpoint, with a backoff, constantly looking for work. -Just not sure that make sense. +Key lifecycle events (start, end, error) are published to Lightning via POST. -I think the execution plan needs to be added to the queue and we read this off once. +Runtime logs are sent to Lightning in batches via a websocket [TBD] -Should a runtime manager be responsible for keeping its own history? I dont think so, I think it should be stateless really. Makes scaling easier. +## Data Structures + +At the time of writing, the Runtime Manager server uses a Lightning Attempt and a Runtime Execution Plan + +This means it doesn't really "own" any of its datatypes - it just maps from one to another. Which is nice and feels appropriate to the architecture. + +The Attempt is of the form { jobs, triggers, nodes }, a reflection of the underlying workflow model. + +The Runtime must be able to convert a string configuration or state into an object (using a callback provided by the server, which calls out ot lightning to resolve the SHA/id). This happens just-in-time, after a workflow starts but before each job. At the time of writing it cannot do this. The CLI may need to implement similar functionality (it would be super cool for the CLI to be able to call out to lightning to fetch a credential, with zero persistence on the client machine). + +Issues: + +- I dunno, I think it's backwards. Lightning shouldn't expose its own internal data stucture in the attempt. Surely the process of creating an attempt means generating an execution plan from the Workflow model? +- Also, Lightning needs to be able to do this conversion ANYWAY to enable users to download a workflow and run it in the CLI. +- Forunately it's gonna be easy to conver (or not) the structures on the way in. So I'll just take a lighting-style attempt for now and we can drop it out later if we want + +## Zero Persistence + +So the RTM server grabs an attempt, it gets removed form the Lightning queue. + +The runtime starts processing the job. + +Then the host machine dies. Or loses connectivity. Or the runtime crashes, or hangs forever. Something BAD happens that we can't control. + +The RTM server is restarted. Memory is wiped. + +Now what? The jobs in progress are lost forever because nothing was persisted. Lightning is sitting waiting for the server to return a result - which it won't. + +Solutions: + +- Lightning has its own timeout. So if a job takes more than 5 minutes, it'll count as failed. It can be manually retried. This is not unreasonable. +- Lightning could maintain a keepalive with the RTM server. When the keepalive dies, Lightning will know which jobs are outstanding. It can re-queue them. + +Other pitfalls: + +- What if the job half finished and posted some information to another server before it died? is re-running harmful? Surely this is up to the job writer to avoid duplicating data. diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index b69b2b546..231e34a27 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -16,6 +16,7 @@ "license": "ISC", "dependencies": { "@koa/router": "^12.0.0", + "@openfn/runtime": "workspace:^0.0.24", "axios": "^1.3.4", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0" diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index ece1f72c0..78bce22c0 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -12,11 +12,13 @@ export const API_PREFIX = `/api/1`; export default (router: Router, state: ServerState) => { // POST attempts/next: + // Removes Attempts from the queue and returns them to the calleer // 200 - return an array of pending attempts // 204 - queue empty (no body) router.post(`${API_PREFIX}/attempts/next`, createFetchNextJob(state)); // GET credential/:id + // Get a credential // 200 - return a credential object // 404 - credential not found router.get(`${API_PREFIX}/credential/:id`, createGetCredential(state)); @@ -24,6 +26,7 @@ export default (router: Router, state: ServerState) => { // Notify of some job update // proxy to event emitter // { event: 'event-name', ...data } + // TODO this should use a websocket to handle the high volume of logs router.post(`${API_PREFIX}/attempts/notify/:id`, createNotify(state)); // Notify an attempt has finished @@ -32,6 +35,10 @@ export default (router: Router, state: ServerState) => { // { data } | { error } router.post(`${API_PREFIX}/attempts/complete/:id`, createComplete(state)); + // TODO i want this too: confirm that an attempt has started + router.post(`${API_PREFIX}/attempts/start/:id`, () => {}); + + // Listing APIs - these list details without changing anything router.get(`${API_PREFIX}/attempts/:id`, unimplemented); router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 2ac02b26d..9767327b0 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -22,9 +22,9 @@ export type RTMEvent = | 'job-start' | 'job-complete' | 'job-log' - | 'job-error' - | 'workflow-start' - | 'workflow-complete' + //| 'job-error' + | 'workflow-start' // before compile + | 'workflow-complete' // after everything has run | 'workflow-error'; // ? export type JobStartEvent = { @@ -56,7 +56,7 @@ let autoServerId = 0; // Before we execute each job (expression), we have to build a state object // This means squashing together the input state and the credential -// The crediential of course is the hard bit +// The credential of course is the hard bit const assembleState = () => {}; // Pre-process a plan diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 0f3e82b45..2cef80089 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -1,47 +1,30 @@ /** * server needs to * - * - create a runtime maager + * - create a runtime manager * - know how to speak to a lightning endpoint to fetch workflows * Is this just a string url? * */ import Koa from 'koa'; -import axios from 'axios'; +import axios from 'axios'; // TODO don't use axios import createAPI from './api'; import startWorkLoop from './work-loop'; -import createMockRTM from './mock/runtime-manager'; - -// This loop will call out to ask for work, with a backoff -const workBackoffLoop = async (lightningUrl: string, rtm: any) => { - let timeout = 100; // TODO strange stuff happens if this has a low value - - const result = await axios.post(`${lightningUrl}/api/1/attempts/next`, { - id: rtm.id, - }); - - if (result.data) { - // TODO handle multiple attempts - const [attempt] = result.data; - rtm.execute(attempt); - } else { - timeout = timeout * 2; - } - setTimeout(() => { - workBackoffLoop(lightningUrl, rtm); - }, timeout); -}; +import convertAttempt from './util/convert-attempt'; +import { Attempt } from './types'; const postResult = async ( lightningUrl: string, attemptId: string, state: any ) => { - const result = await axios.post( - `${lightningUrl}/api/1/attempts/complete/${attemptId}`, - state || {} - ); + if (lightningUrl) { + const result = await axios.post( + `${lightningUrl}/api/1/attempts/complete/${attemptId}`, + state || {} + ); + } // TODO what if result is not 200? // Backoff and try again? }; @@ -54,10 +37,13 @@ type ServerOptions = { rtm?: any; }; -function createServer(options: ServerOptions = {}) { +function createServer(rtm: any, options: ServerOptions = {}) { const app = new Koa(); - const rtm = options.rtm || new createMockRTM(); + const execute = (attempt: Attempt) => { + const plan = convertAttempt(attempt); + rtm.execute(plan); + }; const apiRouter = createAPI(); app.use(apiRouter.routes()); @@ -65,26 +51,29 @@ function createServer(options: ServerOptions = {}) { app.listen(options.port || 1234); - if (options.lightning) { - startWorkLoop(options.lightning, rtm); - - rtm.on('workflow-complete', ({ id, state }) => { - postResult(options.lightning!, id, state); - }); + app.destroy = () => { + // TODO close the work loop + }; - // TODO how about an 'all' so we can "route" events? + if (options.lightning) { + startWorkLoop(options.lightning, rtm.id, execute); } + // TODO how about an 'all' so we can "route" events? + rtm.on('workflow-complete', ({ id, state }) => { + postResult(options.lightning!, id, state); + }); + // TMP doing this for tests but maybe its better done externally app.on = (...args) => rtm.on(...args); app.once = (...args) => rtm.once(...args); - return app; + // debug API to run a workflow + // Used in unit tests + // Only loads in dev mode? + app.execute = execute; - // if not rtm create a mock - // setup routes - // start listening on options.port - // return the server + return app; } export default createServer; diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index d7ffc3628..5831d2347 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -12,60 +12,69 @@ export type State = { [key: string]: any; }; +export type Node = { + id: string; + body?: string; + adaptor?: string; + credential?: any; // TODO tighten this up, string or object + type?: 'webhook' | 'cron'; // trigger only +}; + +export interface Edge { + id: string; + source_job_id?: string; + source_trigger_id?: string; + target_job_id: string; + name?: string; + condition?: string; + error_path?: boolean; + errors?: any; +} + // An attempt object returned by Lightning -// Lightning will build this wfrom a Workflow and Attempt -export type LightningAttempt = { +// We may later drop this abstraction and just accept an excecution plan directly +export type Attempt = { id: string; - input: Omit; // initial state - plan: LightningJob[]; + + triggers: Node[]; + jobs: Node[]; + edges: Edge[]; // these will probably be included by lightning but we don't care here - projectId: string; - status: string; - worker: string; + projectId?: string; + status?: string; + worker?: string; }; -export type LightningJob = { - adaptor: string; - expression: string; // the code we actually want to execute. Could be lazy loaded - credential: id; - - upstream: - | LightningJob // shorthand for { default } - | { - success: LightningJob; - error: LightningJob; - default: LightningJob; - }; -}; +// type RuntimeExecutionPlanID = string; -type RuntimeExecutionPlanID = string; +// type JobEdge = { +// condition?: string; // Javascript expression (function body, not function) +// label?: string; +// }; -// TODO this type should later be imported from the runtime -type JobPlan = { - id: string; - adaptor: string; - expression: string; // the code we actually want to execute. Could be lazy loaded - credential: string | object; // credential can be inline or lazy loaded - state?: Omit; // initial state - - upstream: - | JobPlanID // shorthand for { default } - | { - success: JobPlanID; - error: JobPlanID; - default: JobPlanID; - }; -}; +// // TODO this type should later be imported from the runtime +// // Well, it's not quite the same is it, because eg credential can be a string +// type JobNode = { +// id?: string; -// A runtime manager execution plan -export type ExecutionPlan = { - id: string; // UUID for this plan +// adaptor?: string; - // should we save the initial and resulting status? - // Should we have a status here, is this a living thing? +// expression?: string | Operation[]; // the code we actually want to execute. Can be a path. - plan: JobPlan[]; // TODO this type should later be imported from the runtime -}; +// configuration?: object | string; +// data?: State['data'] | string; // default state (globals) + +// next?: string | Record; +// }; + +// // Note that if the runtime itself is capable of calling an endpoint +// // To fetch credentials (and state?) just-in-time, then this is just a +// // Runtime Exeuction Plan, and we can import it. This is nicer tbh. +// export type ExecutionPlan = { +// id: string; // UUID for this plan + +// start?: JobNodeID; -export type Xplan = ExecutionPlan; +// plan: JobNode[]; // TODO this type should later be imported from the runtime +// }; diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/rtm-server/src/util/convert-attempt.ts new file mode 100644 index 000000000..4becaf5e0 --- /dev/null +++ b/packages/rtm-server/src/util/convert-attempt.ts @@ -0,0 +1,65 @@ +import type { ExecutionPlan, JobNode, JobNodeID } from '@openfn/runtime'; +import { Attempt } from '../types'; + +export default (attempt: Attempt): ExecutionPlan => { + const plan = { + id: attempt.id, + plan: [], + }; + + const nodes: Record = {}; + + const edges = attempt.edges ?? []; + + // We don't really care about triggers, it's mostly just a empty node + if (attempt.triggers?.length) { + attempt.triggers.forEach((trigger) => { + const id = trigger.id || 'trigger'; + + nodes[id] = { + id, + }; + + // TODO do we need to support multiple edges here? Likely + const connectedEdges = edges.filter((e) => e.source_trigger_id === id); + if (connectedEdges.length) { + nodes[id].next = connectedEdges.reduce((obj, edge) => { + obj[edge.target_job_id] = true; + return obj; + }, {}); + } else { + // TODO what if the edge isn't found? + } + }); + } + + if (attempt.jobs?.length) { + attempt.jobs.forEach((job) => { + const id = job.id || 'trigger'; + + nodes[id] = { + id, + configuration: job.credential, // TODO runtime needs to support string credentials + expression: job.body, + adaptor: job.adaptor, + }; + + const next = edges + .filter((e) => e.source_job_id === id) + .reduce((obj, edge) => { + obj[edge.target_job_id] = edge.condition + ? { expression: edge.condition } + : true; + return obj; + }, {}); + + if (Object.keys(next).length) { + nodes[id].next = next; + } + }); + } + + plan.plan = Object.values(nodes); + + return plan; +}; diff --git a/packages/rtm-server/src/util/index.ts b/packages/rtm-server/src/util/index.ts new file mode 100644 index 000000000..48e99fb0c --- /dev/null +++ b/packages/rtm-server/src/util/index.ts @@ -0,0 +1,4 @@ +import convertAttempt from './convert-attempt'; +import { tryWithBackoff } from './try-with-backoff'; + +export { convertAttempt, tryWithBackoff }; diff --git a/packages/rtm-server/src/util.ts b/packages/rtm-server/src/util/try-with-backoff.ts similarity index 94% rename from packages/rtm-server/src/util.ts rename to packages/rtm-server/src/util/try-with-backoff.ts index dd0392ca8..7170a7e99 100644 --- a/packages/rtm-server/src/util.ts +++ b/packages/rtm-server/src/util/try-with-backoff.ts @@ -15,7 +15,7 @@ type Options = { // under what circumstance should this function throw? // If it timesout // Can the inner function force a throw? An exit early? -export const tryWithBackoff = (fn, opts: Options = {}) => { +const tryWithBackoff = (fn, opts: Options = {}) => { if (!opts.timeout) { opts.timeout = 100; // TODO errors occur if this is too low? } @@ -48,3 +48,5 @@ export const tryWithBackoff = (fn, opts: Options = {}) => { } }); }; + +export default tryWithBackoff; diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index cab69c397..3db4b206d 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -1,26 +1,21 @@ -// idk what to call this -// Does it count as part of the API? -// Is it a kind of middleware? -// Is it one of many RTM helpers? - import axios from 'axios'; -import { tryWithBackoff } from './util'; +import tryWithBackoff from './util/try-with-backoff'; +import { Attempt } from './types'; // TODO how does this report errors, like if Lightning is down? // Or auth is bad? -export default (lightningUrl: string, rtm: any) => { +export default ( + lightningUrl: string, + rtmId: string, + execute: (attempt: Attempt) => void +) => { const fetchWork = async () => { // TODO what if this retuns like a 500? Server down? const result = await axios.post(`${lightningUrl}/api/1/attempts/next`, { - id: rtm.id, + id: rtmId, }); if (result.data) { - // TODO handle multiple attempts - const [attempt] = result.data; - // Start the job (but don't wtit for it) - rtm.execute(attempt); - - // Return true t + result.data.forEach(execute); return true; } // return false to backoff and try again diff --git a/packages/rtm-server/test/events.test.ts b/packages/rtm-server/test/events.test.ts new file mode 100644 index 000000000..bdb7bbd82 --- /dev/null +++ b/packages/rtm-server/test/events.test.ts @@ -0,0 +1,53 @@ +/** + * Unit tests on events published by the rtm-server + * No lightning involved here + */ +import test from 'ava'; +import createRTMServer from '../src/server'; +import createMockRTM from '../src/mock/runtime-manager'; +import { waitForEvent } from './util'; + +let server; + +test.before(() => { + const rtm = createMockRTM(); + server = createRTMServer(rtm, { port: 2626 }); +}); + +test.serial( + 'trigger a workflow-start event when execution starts', + async (t) => { + server.execute({ + id: 'a', + triggers: [{ id: 't', next: { b: true } }], + jobs: [{ id: 'j' }], + }); + + const evt = await waitForEvent(server, 'workflow-start'); + t.truthy(evt); + t.is(evt.id, 'a'); + } +); + +test.serial.skip('should pick up a novel attempt in the queue', async (t) => { + lng.addToQueue({ id: 'x', plan: [{ expression: '{}' }] }); + const evt = await waitForEvent(rtm, 'workflow-start'); + t.truthy(evt); + t.is(evt.id, 'x'); + + // let the workflow finish processing + await waitForEvent(rtm, 'workflow-complete'); +}); + +test.serial.skip( + 'should publish a workflow-complete event with state', + async (t) => { + // The mock RTM will evaluate the expression as JSON and return it + lng.addToQueue({ id: 'x', plan: [{ expression: '{ "answer": 42 }' }] }); + + const evt = await waitForEvent(rtm, 'workflow-complete'); + t.truthy(evt); + t.is(evt.id, 'x'); + t.deepEqual(evt.state, { answer: 42 }); + } +); diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 0f9cff90b..77079a369 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -1,5 +1,7 @@ -// these integration tests test the real rtm server logic with a mock lightning and a mock rtm -// So maybe it's not really "integration" tests after all, but regular server tests +/* + * Tests of Lightning-RTM server integration, from Lightning's perspective + */ + import test from 'ava'; import createRTMServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; @@ -19,35 +21,19 @@ test.before(() => { rtm = createRTMServer({ port: 4567, lightning: urls.lng }); }); -test.serial('should pick up a default attempt in the queue', async (t) => { - lng.addToQueue('attempt-1'); - const evt = await waitForEvent(rtm, 'workflow-start'); - t.truthy(evt); - t.is(evt.id, 'attempt-1'); -}); - -test.serial('should pick up a novel attempt in the queue', async (t) => { - lng.addToQueue({ id: 'x', plan: [{ expression: '{}' }] }); - const evt = await waitForEvent(rtm, 'workflow-start'); - t.truthy(evt); - t.is(evt.id, 'x'); +// Really high level test +test.serial('process an attempt', async (t) => { + lng.addAttempt('a1', { + // workflow goes here + }); - // let the workflow finish processing - await waitForEvent(rtm, 'workflow-complete'); + lng.waitForResult('a1', (result) => { + // test the result here + t.is(result.answer, 42); + }); }); -test.serial( - 'should publish a workflow-complete event with state', - async (t) => { - // The mock RTM will evaluate the expression as JSON and return it - lng.addToQueue({ id: 'x', plan: [{ expression: '{ "answer": 42 }' }] }); - - const evt = await waitForEvent(rtm, 'workflow-complete'); - t.truthy(evt); - t.is(evt.id, 'x'); - t.deepEqual(evt.state, { answer: 42 }); - } -); +// process multple attempts test.serial( 'should post to attempts/complete with the final state', diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index cf71a5d07..495609eed 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -5,8 +5,8 @@ import create, { WorkflowCompleteEvent, WorkflowStartEvent, } from '../../src/mock/runtime-manager'; -import { ExecutionPlan } from '../../src/types'; -import { waitForEvent, clone } from '../util'; +import type { ExecutionPlan } from '@openfn/runtime'; +import { waitForEvent, clone } from '../util'; // ??? const sampleWorkflow = { id: 'w1', diff --git a/packages/rtm-server/test/util/convert-attempt.test.ts b/packages/rtm-server/test/util/convert-attempt.test.ts new file mode 100644 index 000000000..956ed6505 --- /dev/null +++ b/packages/rtm-server/test/util/convert-attempt.test.ts @@ -0,0 +1,222 @@ +import test from 'ava'; +import convertAttempt from '../../src/util/convert-attempt'; +import { Attempt, Node } from '../../src/types'; + +// Creates a lightning node (job or trigger) +const createNode = (props = {}) => + ({ + id: 'a', + body: 'x', + adaptor: 'common', + credential: 'y', + ...props, + } as Node); + +const createEdge = (from, to, props = {}) => ({ + id: `${from}-${to}`, + source_job_id: from, + target_job_id: to, + ...props, +}); + +// Creates a lightning trigger +const createTrigger = (props = {}) => + ({ + id: 't', + type: 'cron', + ...props, + } as Node); + +// Creates a runtime job node +const createJob = (props = {}) => ({ + id: 'a', + expression: 'x', + adaptor: 'common', + configuration: 'y', + ...props, +}); + +test('convert a single job', (t) => { + const attempt: Attempt = { + id: 'w', + jobs: [createNode()], + triggers: [], + edges: [], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [createJob()], + }); +}); + +test('Accept a partial attempt object', (t) => { + const attempt: Attempt = { + id: 'w', + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [], + }); +}); + +test('convert a single trigger', (t) => { + const attempt: Attempt = { + id: 'w', + triggers: [createTrigger()], + jobs: [], + edges: [], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [ + { + id: 't', + }, + ], + }); +}); + +// This exhibits current behaviour. This should never happen though +test('ignore a single edge', (t) => { + const attempt: Attempt = { + id: 'w', + jobs: [], + triggers: [], + edges: [createEdge('a', 'b')], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [], + }); +}); + +test('convert a single trigger with an edge', (t) => { + const attempt: Attempt = { + id: 'w', + triggers: [createTrigger()], + jobs: [createNode()], + edges: [ + { + id: 'w-t', + source_trigger_id: 't', + target_job_id: 'a', + }, + ], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [ + { + id: 't', + next: { + a: true, + }, + }, + createJob(), + ], + }); +}); + +test('convert a single trigger with two edges', (t) => { + const attempt: Attempt = { + id: 'w', + triggers: [createTrigger()], + jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], + edges: [ + { + id: 't-a', + source_trigger_id: 't', + target_job_id: 'a', + }, + { + id: 't-b', + source_trigger_id: 't', + target_job_id: 'b', + }, + ], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [ + { + id: 't', + next: { + a: true, + b: true, + }, + }, + createJob({ id: 'a' }), + createJob({ id: 'b' }), + ], + }); +}); + +test('convert two linked jobs', (t) => { + const attempt: Attempt = { + id: 'w', + jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], + triggers: [], + edges: [createEdge('a', 'b')], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [createJob({ id: 'a', next: { b: true } }), createJob({ id: 'b' })], + }); +}); + +// This isn't supported by the runtime, but it'll survive the conversion +test('convert a job with two upstream jobs', (t) => { + const attempt: Attempt = { + id: 'w', + jobs: [ + createNode({ id: 'a' }), + createNode({ id: 'b' }), + createNode({ id: 'x' }), + ], + triggers: [], + edges: [createEdge('a', 'x'), createEdge('b', 'x')], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [ + createJob({ id: 'a', next: { x: true } }), + createJob({ id: 'b', next: { x: true } }), + createJob({ id: 'x' }), + ], + }); +}); + +test('convert two linked jobs with an edge condition', (t) => { + const condition = 'state.age > 10'; + const attempt: Attempt = { + id: 'w', + jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], + triggers: [], + edges: [createEdge('a', 'b', { condition })], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + plan: [ + createJob({ id: 'a', next: { b: { expression: condition } } }), + createJob({ id: 'b' }), + ], + }); +}); From a5943afb82d620cc23f4d32e968952719a58c331 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 26 May 2023 14:50:40 +0100 Subject: [PATCH 012/232] rtm-server: fix all tests Apart from integration, which is gona have a rethink --- packages/rtm-server/package.json | 2 +- .../rtm-server/src/mock/runtime-manager.ts | 34 ++++++++----------- .../rtm-server/src/util/convert-attempt.ts | 7 ++-- packages/rtm-server/src/util/index.ts | 2 +- packages/rtm-server/test/events.test.ts | 30 ++++++++-------- packages/rtm-server/test/integration.test.ts | 7 ++-- .../test/mock/runtime-manager.test.ts | 6 ++-- packages/rtm-server/test/server.test.ts | 5 ++- .../test/util/convert-attempt.test.ts | 22 ++++++------ .../test/util/try-with-backoff.test.ts | 2 +- pnpm-lock.yaml | 3 ++ 11 files changed, 60 insertions(+), 60 deletions(-) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 231e34a27..4c6150587 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -16,7 +16,7 @@ "license": "ISC", "dependencies": { "@koa/router": "^12.0.0", - "@openfn/runtime": "workspace:^0.0.24", + "@openfn/runtime": "workspace:*", "axios": "^1.3.4", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0" diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 9767327b0..bdbb2888b 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'node:events'; +import type { ExecutionPlan } from '@openfn/runtime'; -import type { State, Credential, ExecutionPlan, JobPlan } from '../types'; +import type { State, Credential } from '../types'; import mockResolvers from './resolvers'; // A mock runtime manager @@ -59,11 +60,6 @@ let autoServerId = 0; // The credential of course is the hard bit const assembleState = () => {}; -// Pre-process a plan -// Validate it -// Assign ids to jobs if they don't exist -const preprocessPlan = (plan: ExecutionPlan) => plan; - function createMock( serverId = autoServerId, resolvers: LazyResolvers = mockResolvers @@ -85,14 +81,13 @@ function createMock( bus.once(event, fn); }; - const executeJob = async (job: JobPlan) => { + const executeJob = async (job: JobPlan, initialState = {}) => { // TODO maybe lazy load the job from an id - const { id, expression, credential } = job; - - if (typeof credential === 'string') { + const { id, expression, configuration } = job; + if (typeof configuration === 'string') { // Fetch the credntial but do nothing with it // Maybe later we use it to assemble state - await resolvers.credentials(credential); + await resolvers.credentials(configuration); } // Does a job reallly need its own id...? Maybe. @@ -108,7 +103,7 @@ function createMock( // It's the json output of the logger dispatch('job-log', { id, runId }); - let state = {}; + let state = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { state = JSON.parse(expression); @@ -124,17 +119,18 @@ function createMock( // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = (xplan: ExecutionPlan) => { - const { id, plan } = preprocessPlan(xplan); - activeWorkflows[id] = true; + const { id, jobs } = xplan; + activeWorkflows[id!] = true; setTimeout(async () => { dispatch('workflow-start', { id }); setTimeout(async () => { - // Run the first job - // TODO this need sto loop over all jobs - const state = await executeJob(plan[0]); - + let state = {}; + // Trivial job reducer in our mock + for (const job of jobs) { + state = await executeJob(job, state); + } setTimeout(() => { - delete activeWorkflows[id]; + delete activeWorkflows[id!]; dispatch('workflow-complete', { id, state }); }, 1); }, 1); diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/rtm-server/src/util/convert-attempt.ts index 4becaf5e0..be86b2af3 100644 --- a/packages/rtm-server/src/util/convert-attempt.ts +++ b/packages/rtm-server/src/util/convert-attempt.ts @@ -2,9 +2,8 @@ import type { ExecutionPlan, JobNode, JobNodeID } from '@openfn/runtime'; import { Attempt } from '../types'; export default (attempt: Attempt): ExecutionPlan => { - const plan = { + const plan: Partial = { id: attempt.id, - plan: [], }; const nodes: Record = {}; @@ -59,7 +58,7 @@ export default (attempt: Attempt): ExecutionPlan => { }); } - plan.plan = Object.values(nodes); + plan.jobs = Object.values(nodes); - return plan; + return plan as ExecutionPlan; }; diff --git a/packages/rtm-server/src/util/index.ts b/packages/rtm-server/src/util/index.ts index 48e99fb0c..1a186e17a 100644 --- a/packages/rtm-server/src/util/index.ts +++ b/packages/rtm-server/src/util/index.ts @@ -1,4 +1,4 @@ import convertAttempt from './convert-attempt'; -import { tryWithBackoff } from './try-with-backoff'; +import tryWithBackoff from './try-with-backoff'; export { convertAttempt, tryWithBackoff }; diff --git a/packages/rtm-server/test/events.test.ts b/packages/rtm-server/test/events.test.ts index bdb7bbd82..5678e7dc6 100644 --- a/packages/rtm-server/test/events.test.ts +++ b/packages/rtm-server/test/events.test.ts @@ -7,6 +7,8 @@ import createRTMServer from '../src/server'; import createMockRTM from '../src/mock/runtime-manager'; import { waitForEvent } from './util'; +const str = (obj: object) => JSON.stringify(obj); + let server; test.before(() => { @@ -26,28 +28,24 @@ test.serial( const evt = await waitForEvent(server, 'workflow-start'); t.truthy(evt); t.is(evt.id, 'a'); + + // TODO what goes in this event? + // Test more carefully } ); -test.serial.skip('should pick up a novel attempt in the queue', async (t) => { - lng.addToQueue({ id: 'x', plan: [{ expression: '{}' }] }); - const evt = await waitForEvent(rtm, 'workflow-start'); - t.truthy(evt); - t.is(evt.id, 'x'); - - // let the workflow finish processing - await waitForEvent(rtm, 'workflow-complete'); -}); - -test.serial.skip( - 'should publish a workflow-complete event with state', +test.serial.only( + 'trigger a workflow-complete event when execution completes', async (t) => { - // The mock RTM will evaluate the expression as JSON and return it - lng.addToQueue({ id: 'x', plan: [{ expression: '{ "answer": 42 }' }] }); + server.execute({ + id: 'a', + triggers: [{ id: 't', next: { b: true } }], + jobs: [{ id: 'j', body: str({ answer: 42 }) }], + }); - const evt = await waitForEvent(rtm, 'workflow-complete'); + const evt = await waitForEvent(server, 'workflow-complete'); t.truthy(evt); - t.is(evt.id, 'x'); + t.is(evt.id, 'a'); t.deepEqual(evt.state, { answer: 42 }); } ); diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 77079a369..47ec2b599 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -5,6 +5,7 @@ import test from 'ava'; import createRTMServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; +import createMockRTM from '../src/mock/runtime-manager'; import { wait, waitForEvent } from './util'; @@ -18,11 +19,11 @@ const urls = { test.before(() => { lng = createLightningServer({ port: 7654 }); - rtm = createRTMServer({ port: 4567, lightning: urls.lng }); + rtm = createRTMServer(createMockRTM(), { port: 4567, lightning: urls.lng }); }); // Really high level test -test.serial('process an attempt', async (t) => { +test.serial.skip('process an attempt', async (t) => { lng.addAttempt('a1', { // workflow goes here }); @@ -35,7 +36,7 @@ test.serial('process an attempt', async (t) => { // process multple attempts -test.serial( +test.serial.skip( 'should post to attempts/complete with the final state', async (t) => { // The mock RTM will evaluate the expression as JSON and return it diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index 495609eed..761119a32 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -10,7 +10,7 @@ import { waitForEvent, clone } from '../util'; // ??? const sampleWorkflow = { id: 'w1', - plan: [ + jobs: [ { id: 'j1', adaptor: 'common@1.0.0', @@ -30,7 +30,7 @@ test('mock runtime manager should have an id', (t) => { }); test('getStatus() should should have no active workflows', (t) => { - const rtm = create(22); + const rtm = create(); const { active } = rtm.getStatus(); t.is(active, 0); @@ -102,7 +102,7 @@ test('mock should evaluate expressions as JSON', async (t) => { test('resolve credential before job-start if credential is a string', async (t) => { const wf = clone(sampleWorkflow); - wf.plan[0].credential = 'x'; + wf.jobs[0].configuration = 'x'; let didCallCredentials; const credentials = async (_id) => { diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index f1a9dbd92..270df2c1d 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,6 +1,8 @@ import test from 'ava'; import axios from 'axios'; + import createServer from '../src/server'; +import createMockRTM from '../src/mock/runtime-manager'; // Unit tests against the RTM web server // I don't think there will ever be much here because the server is mostly a pull @@ -10,7 +12,8 @@ let server; const url = 'http://localhost:7777'; test.beforeEach(() => { - server = createServer({ port: 7777 }); + const rtm = createMockRTM(); + server = createServer(rtm, { port: 7777 }); }); test('healthcheck', async (t) => { diff --git a/packages/rtm-server/test/util/convert-attempt.test.ts b/packages/rtm-server/test/util/convert-attempt.test.ts index 956ed6505..adb94e506 100644 --- a/packages/rtm-server/test/util/convert-attempt.test.ts +++ b/packages/rtm-server/test/util/convert-attempt.test.ts @@ -47,19 +47,19 @@ test('convert a single job', (t) => { t.deepEqual(result, { id: 'w', - plan: [createJob()], + jobs: [createJob()], }); }); test('Accept a partial attempt object', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', - plan: [], + jobs: [], }); }); @@ -74,7 +74,7 @@ test('convert a single trigger', (t) => { t.deepEqual(result, { id: 'w', - plan: [ + jobs: [ { id: 't', }, @@ -94,7 +94,7 @@ test('ignore a single edge', (t) => { t.deepEqual(result, { id: 'w', - plan: [], + jobs: [], }); }); @@ -115,7 +115,7 @@ test('convert a single trigger with an edge', (t) => { t.deepEqual(result, { id: 'w', - plan: [ + jobs: [ { id: 't', next: { @@ -149,7 +149,7 @@ test('convert a single trigger with two edges', (t) => { t.deepEqual(result, { id: 'w', - plan: [ + jobs: [ { id: 't', next: { @@ -174,7 +174,7 @@ test('convert two linked jobs', (t) => { t.deepEqual(result, { id: 'w', - plan: [createJob({ id: 'a', next: { b: true } }), createJob({ id: 'b' })], + jobs: [createJob({ id: 'a', next: { b: true } }), createJob({ id: 'b' })], }); }); @@ -194,7 +194,7 @@ test('convert a job with two upstream jobs', (t) => { t.deepEqual(result, { id: 'w', - plan: [ + jobs: [ createJob({ id: 'a', next: { x: true } }), createJob({ id: 'b', next: { x: true } }), createJob({ id: 'x' }), @@ -214,7 +214,7 @@ test('convert two linked jobs with an edge condition', (t) => { t.deepEqual(result, { id: 'w', - plan: [ + jobs: [ createJob({ id: 'a', next: { b: { expression: condition } } }), createJob({ id: 'b' }), ], diff --git a/packages/rtm-server/test/util/try-with-backoff.test.ts b/packages/rtm-server/test/util/try-with-backoff.test.ts index 8516e1e40..b0c864567 100644 --- a/packages/rtm-server/test/util/try-with-backoff.test.ts +++ b/packages/rtm-server/test/util/try-with-backoff.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; -import { tryWithBackoff } from '../../src/util'; +import tryWithBackoff from '../../src/util/try-with-backoff'; test('return immediately', async (t) => { let callCount = 0; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f9f93483..6425a0ada 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -329,6 +329,9 @@ importers: '@koa/router': specifier: ^12.0.0 version: 12.0.0 + '@openfn/runtime': + specifier: workspace:* + version: link:../runtime axios: specifier: ^1.3.4 version: 1.3.4 From 3c2349f2bbbe976569fee325476ed4f064097446 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 26 May 2023 16:23:47 +0100 Subject: [PATCH 013/232] rtm-server: remove axios --- packages/rtm-server/package.json | 3 +- .../rtm-server/src/mock/lightning/server.ts | 1 + packages/rtm-server/src/server.ts | 8 +- packages/rtm-server/src/work-loop.ts | 6 +- .../rtm-server/test/mock/lightning.test.ts | 111 ++++++++++-------- packages/rtm-server/test/server.test.ts | 6 +- pnpm-lock.yaml | 19 +-- 7 files changed, 82 insertions(+), 72 deletions(-) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 4c6150587..b332ecf10 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -6,7 +6,7 @@ "type": "module", "private": true, "scripts": { - "test": "pnpm ava", + "test": "pnpm ava --serial", "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", @@ -17,7 +17,6 @@ "dependencies": { "@koa/router": "^12.0.0", "@openfn/runtime": "workspace:*", - "axios": "^1.3.4", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0" }, diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 1444546d4..3c8609077 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -60,6 +60,7 @@ const createLightningServer = (options = {}) => { return true; } }; + app.resetQueue = () => (state.queue = []); app.getQueueLength = () => state.queue.length; app.getResult = (attemptId: string) => state.results[attemptId]; app.on = (event: 'notify', fn: (evt: any) => void) => { diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 2cef80089..ce79103bc 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -8,7 +8,6 @@ */ import Koa from 'koa'; -import axios from 'axios'; // TODO don't use axios import createAPI from './api'; import startWorkLoop from './work-loop'; import convertAttempt from './util/convert-attempt'; @@ -20,9 +19,12 @@ const postResult = async ( state: any ) => { if (lightningUrl) { - const result = await axios.post( + const result = await fetch( `${lightningUrl}/api/1/attempts/complete/${attemptId}`, - state || {} + { + method: 'POST', + body: state || {}, + } ); } // TODO what if result is not 200? diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index 3db4b206d..cc03cff7b 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -1,4 +1,3 @@ -import axios from 'axios'; import tryWithBackoff from './util/try-with-backoff'; import { Attempt } from './types'; @@ -11,8 +10,9 @@ export default ( ) => { const fetchWork = async () => { // TODO what if this retuns like a 500? Server down? - const result = await axios.post(`${lightningUrl}/api/1/attempts/next`, { - id: rtmId, + const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { + method: 'POST', + body: { id: rtmId }, }); if (result.data) { result.data.forEach(execute); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 0f7ee0ec5..a6664bbeb 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,5 +1,4 @@ import test from 'ava'; -import axios from 'axios'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; @@ -7,21 +6,34 @@ const baseUrl = `http://localhost:8888${API_PREFIX}`; let server; -test.beforeEach(() => { +test.before(() => { server = createLightningServer({ port: 8888 }); }); test.afterEach(() => { + server.resetQueue(); +}); + +test.after(() => { server.destroy(); }); -const get = (path: string) => axios.get(`${baseUrl}/${path}`); +const get = (path: string) => fetch(`${baseUrl}/${path}`); const post = (path: string, data: any) => - axios.post(`${baseUrl}/${path}`, data); + fetch(`${baseUrl}/${path}`, { + method: 'POST', + body: JSON.stringify(data), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); test.serial('GET /credential - return a credential', async (t) => { - const { status, data: job } = await get('credential/a'); - t.is(status, 200); + const res = await get('credential/a'); + t.is(res.status, 200); + + const job = await res.json(); t.is(job.user, 'bobby'); t.is(job.password, 'password1'); @@ -29,8 +41,11 @@ test.serial('GET /credential - return a credential', async (t) => { test.serial('GET /credential - return a new mock credential', async (t) => { server.addCredential('b', { user: 'johnny', password: 'cash' }); - const { status, data: job } = await get('credential/b'); - t.is(status, 200); + + const res = await get('credential/b'); + t.is(res.status, 200); + + const job = await res.json(); t.is(job.user, 'johnny'); t.is(job.password, 'cash'); @@ -39,45 +54,40 @@ test.serial('GET /credential - return a new mock credential', async (t) => { test.serial( 'GET /credential - return 404 if no credential found', async (t) => { - try { - await get('credential/c'); - } catch (e) { - t.is(e.response.status, 404); - } + const res = await get('credential/c'); + t.is(res.status, 404); } ); test.serial( - 'POST /attempts/next - return 204 for an empty queue', + 'POST /attempts/next - return 204 and no body for an empty queue', async (t) => { t.is(server.getQueueLength(), 0); - const { status, data } = await post('attempts/next', { id: 'x' }); - t.is(status, 204); - - t.falsy(data); + const res = await post('attempts/next', { id: 'x' }); + t.is(res.status, 204); + t.false(res.bodyUsed); } ); test.serial('POST /attempts/next - return 400 if no id provided', async (t) => { - try { - await post('attempts/next', {}); - } catch (e) { - t.is(e.response.status, 400); - } + const res = await post('attempts/next', {}); + t.is(res.status, 400); }); test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { server.addToQueue('attempt-1'); t.is(server.getQueueLength(), 1); - const { status, data } = await post('attempts/next', { id: 'x' }); - t.is(status, 200); - t.truthy(data); - t.true(Array.isArray(data)); - t.is(data.length, 1); + const res = await post('attempts/next', { id: 'x' }); + const result = await res.json(); + t.is(res.status, 200); + + t.truthy(result); + t.true(Array.isArray(result)); + t.is(result.length, 1); // not interested in testing much against the attempt structure at this stage - const [attempt] = data; + const [attempt] = result; t.is(attempt.id, 'attempt-1'); t.true(Array.isArray(attempt.plan)); @@ -89,14 +99,16 @@ test.serial( async (t) => { server.addToQueue({ id: 'abc' }); t.is(server.getQueueLength(), 1); - const { status, data } = await post('attempts/next', { id: 'x' }); - t.is(status, 200); - t.truthy(data); - t.true(Array.isArray(data)); - t.is(data.length, 1); + const res = await post('attempts/next', { id: 'x' }); + t.is(res.status, 200); + + const result = await res.json(); + t.truthy(result); + t.true(Array.isArray(result)); + t.is(result.length, 1); - const [attempt] = data; + const [attempt] = result; t.is(attempt.id, 'abc'); t.is(server.getQueueLength(), 0); @@ -108,12 +120,14 @@ test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { server.addToQueue('attempt-1'); server.addToQueue('attempt-1'); t.is(server.getQueueLength(), 3); - const { status, data } = await post('attempts/next?count=2', { id: 'x' }); - t.is(status, 200); - t.truthy(data); - t.true(Array.isArray(data)); - t.is(data.length, 2); + const res = await post('attempts/next?count=2', { id: 'x' }); + t.is(res.status, 200); + + const result = await res.json(); + t.truthy(result); + t.true(Array.isArray(result)); + t.is(result.length, 2); t.is(server.getQueueLength(), 1); }); @@ -122,17 +136,18 @@ test.serial( 'POST /attempts/next - clear the queue after a request', async (t) => { server.addToQueue('attempt-1'); - const req1 = await post('attempts/next', { id: 'x' }); - t.is(req1.status, 200); - - t.is(req1.data.length, 1); - - const req2 = await post('attempts/next', { id: 'x' }); - t.is(req2.status, 204); - t.falsy(req2.data); + const res1 = await post('attempts/next', { id: 'x' }); + t.is(res1.status, 200); + + const result1 = await res1.json(); + t.is(result1.length, 1); + const res2 = await post('attempts/next', { id: 'x' }); + t.is(res2.status, 204); + t.falsy(res2.bodyUsed); } ); +// TODO this API is gonna be restructured test.serial('POST /attempts/notify - should return 200', async (t) => { const { status } = await post('attempts/notify/a', {}); t.is(status, 200); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index 270df2c1d..b115cd564 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,5 +1,4 @@ import test from 'ava'; -import axios from 'axios'; import createServer from '../src/server'; import createMockRTM from '../src/mock/runtime-manager'; @@ -17,7 +16,8 @@ test.beforeEach(() => { }); test('healthcheck', async (t) => { - const result = await axios.get(`${url}/healthcheck`); + const result = await fetch(`${url}/healthcheck`); t.is(result.status, 200); - t.is(result.data, 'OK'); + const body = await result.text(); + t.is(body, 'OK'); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6425a0ada..8e5772518 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -332,9 +332,6 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime - axios: - specifier: ^1.3.4 - version: 1.3.4 koa: specifier: ^2.13.4 version: 2.13.4 @@ -1630,6 +1627,7 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -1705,16 +1703,6 @@ packages: - debug dev: true - /axios@1.3.4: - resolution: {integrity: sha512-toYm+Bsyl6VC5wSkfkbbNB6ROv7KY93PEBBL6xyDczaIHasAiv4wPqQ/c4RjoQzipxRD2W5g21cOqQulZ7rHwQ==} - dependencies: - follow-redirects: 1.15.2 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - dev: false - /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} dev: true @@ -2111,6 +2099,7 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 + dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -2389,6 +2378,7 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3304,6 +3294,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -3317,6 +3308,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -5050,6 +5042,7 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} From bcec14eb3852f82b5d9b3970c3c1c07c4380a307 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 30 May 2023 15:49:41 +0100 Subject: [PATCH 014/232] rtm-server: fix integration --- packages/rtm-server/notes | 15 +++++++ packages/rtm-server/src/mock/data.ts | 2 + packages/rtm-server/src/mock/lightning/api.ts | 8 +++- .../src/mock/lightning/middleware.ts | 3 +- .../rtm-server/src/mock/lightning/server.ts | 29 ++++++++---- .../rtm-server/src/mock/runtime-manager.ts | 2 +- packages/rtm-server/src/server.ts | 6 ++- packages/rtm-server/src/work-loop.ts | 12 +++-- packages/rtm-server/test/integration.test.ts | 18 +++++--- .../rtm-server/test/mock/lightning.test.ts | 44 ++++++++----------- 10 files changed, 90 insertions(+), 49 deletions(-) diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes index 4a5d1ca78..66c705564 100644 --- a/packages/rtm-server/notes +++ b/packages/rtm-server/notes @@ -45,8 +45,23 @@ Now what? The jobs in progress are lost forever because nothing was persisted. L Solutions: - Lightning has its own timeout. So if a job takes more than 5 minutes, it'll count as failed. It can be manually retried. This is not unreasonable. +- What if a RTM job returns after 10 minutes? Lightning probably needs to reject it. It actually needs to handshake - Lightning could maintain a keepalive with the RTM server. When the keepalive dies, Lightning will know which jobs are outstanding. It can re-queue them. Other pitfalls: - What if the job half finished and posted some information to another server before it died? is re-running harmful? Surely this is up to the job writer to avoid duplicating data. + +## Logging + +The RTM will publish a lot of logs. + +Usually, we can just return all the logs when they're done in batch. + +If someone happens to be watching, we may want to stream logs live. This will happen but less frequently. + +Should lightning and the RTM server maintain a web socket to pipe all logs through? + +Should logs be posted in batches? Every n-ms? + +I'm gonna keep it really simple for now, and send all the logs after completion in a single post. diff --git a/packages/rtm-server/src/mock/data.ts b/packages/rtm-server/src/mock/data.ts index f0172a81d..d3254c325 100644 --- a/packages/rtm-server/src/mock/data.ts +++ b/packages/rtm-server/src/mock/data.ts @@ -1,3 +1,5 @@ +// TODO this file, if we really need it, should move into test + export const credentials = () => ({ a: { user: 'bobby', diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 78bce22c0..667d05ab1 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -11,8 +11,12 @@ import type { ServerState } from './server'; export const API_PREFIX = `/api/1`; export default (router: Router, state: ServerState) => { + // Basically all requests must include an rtm id + // And probably later a security token + // POST attempts/next: - // Removes Attempts from the queue and returns them to the calleer + // Removes Attempts from the queue and returns them to the caller + // Lightning should track who has each attempt // 200 - return an array of pending attempts // 204 - queue empty (no body) router.post(`${API_PREFIX}/attempts/next`, createFetchNextJob(state)); @@ -31,8 +35,10 @@ export default (router: Router, state: ServerState) => { // Notify an attempt has finished // Could be error or success state + // If a complete comes in from an unexpected source (ie a timed out job), this should throw // Error or state in payload // { data } | { error } + // TODO result needs to be { rtmId, state, meta } (meta to come, but timing, memory info etc) router.post(`${API_PREFIX}/attempts/complete/:id`, createComplete(state)); // TODO i want this too: confirm that an attempt has started diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index c86322f09..096c8cf18 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -33,7 +33,6 @@ export const createFetchNextJob = payload.push(queue.shift()); count -= 1; } - if (payload.length > 0) { ctx.body = JSON.stringify(payload); ctx.status = 200; @@ -75,7 +74,7 @@ export const createComplete = (state: ServerState) => (ctx: Koa.Context) => { const finalState = ctx.request.body as State; results[ctx.params.id] = finalState; - events.emit('complete', { id: ctx.params.id, state: finalState }); + events.emit('workflow-complete', { id: ctx.params.id, state: finalState }); ctx.status = 200; }; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 3c8609077..6c162719d 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -4,8 +4,7 @@ import bodyParser from 'koa-bodyparser'; import Router from '@koa/router'; import createAPI from './api'; -import * as data from '../data'; -import { LightningAttempt } from '../../types'; +import { Attempt } from '../../types'; import { RTMEvent } from '../runtime-manager'; type NotifyEvent = { @@ -17,7 +16,7 @@ type NotifyEvent = { export type ServerState = { credentials: Record; attempts: Record; - queue: LightningAttempt[]; + queue: Attempt[]; results: Record; events: EventEmitter; }; @@ -26,9 +25,12 @@ export type ServerState = { const createLightningServer = (options = {}) => { // App state const state = { - credentials: data.credentials(), - attempts: data.attempts(), - queue: [] as LightningAttempt[], + credentials: {}, + attempts: [], + + // TODO for now, the queue will hold the actual Attempt data directly + // I think later we want it to just hold an id? + queue: [] as Attempt[], results: {}, events: new EventEmitter(), } as ServerState; @@ -45,10 +47,10 @@ const createLightningServer = (options = {}) => { app.addCredential = (id: string, cred: Credential) => { state.credentials[id] = cred; }; - app.addAttempt = (attempt: LightningAttempt) => { + app.addAttempt = (attempt: Attempt) => { state.attempts[attempt.id] = attempt; }; - app.addToQueue = (attempt: string | LightningAttempt) => { + app.addToQueue = (attempt: string | Attempt) => { if (typeof attempt == 'string') { if (state.attempts[attempt]) { state.queue.push(state.attempts[attempt]); @@ -60,6 +62,17 @@ const createLightningServer = (options = {}) => { return true; } }; + app.waitForResult = (workflowId: string) => { + return new Promise((resolve) => { + const handler = (evt) => { + if (evt.id === workflowId) { + state.events.removeListener('workflow-complete', handler); + resolve(evt); + } + }; + state.events.addListener('workflow-complete', handler); + }); + }; app.resetQueue = () => (state.queue = []); app.getQueueLength = () => state.queue.length; app.getResult = (attemptId: string) => state.results[attemptId]; diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index bdbb2888b..25a756cc7 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -121,7 +121,7 @@ function createMock( const execute = (xplan: ExecutionPlan) => { const { id, jobs } = xplan; activeWorkflows[id!] = true; - setTimeout(async () => { + setTimeout(() => { dispatch('workflow-start', { id }); setTimeout(async () => { let state = {}; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index ce79103bc..be126157a 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -23,7 +23,11 @@ const postResult = async ( `${lightningUrl}/api/1/attempts/complete/${attemptId}`, { method: 'POST', - body: state || {}, + body: JSON.stringify(state || {}), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, } ); } diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index cc03cff7b..3f1f30e68 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -12,10 +12,16 @@ export default ( // TODO what if this retuns like a 500? Server down? const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { method: 'POST', - body: { id: rtmId }, + body: JSON.stringify({ id: rtmId }), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, }); - if (result.data) { - result.data.forEach(execute); + if (result.body) { + result.json().then((workflows) => { + workflows.forEach(execute); + }); return true; } // return false to backoff and try again diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 47ec2b599..857b42840 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -23,15 +23,19 @@ test.before(() => { }); // Really high level test -test.serial.skip('process an attempt', async (t) => { - lng.addAttempt('a1', { - // workflow goes here +test.serial('process an attempt', async (t) => { + lng.addToQueue({ + id: 'a1', + jobs: [ + { + adaptor: '@openfn/language-common@1.0.0', + body: JSON.stringify({ answer: 42 }), + }, + ], }); - lng.waitForResult('a1', (result) => { - // test the result here - t.is(result.answer, 42); - }); + const { state } = await lng.waitForResult('a1'); + t.is(state.answer, 42); }); // process multple attempts diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index a6664bbeb..07c56fd04 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; - +import { attempts } from '../../src/mock/data'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; const baseUrl = `http://localhost:8888${API_PREFIX}`; @@ -29,20 +29,20 @@ const post = (path: string, data: any) => }, }); -test.serial('GET /credential - return a credential', async (t) => { - const res = await get('credential/a'); - t.is(res.status, 200); - - const job = await res.json(); +const attempt1 = attempts()['attempt-1']; - t.is(job.user, 'bobby'); - t.is(job.password, 'password1'); -}); +test.serial( + 'GET /credential - return 404 if no credential found', + async (t) => { + const res = await get('credential/x'); + t.is(res.status, 404); + } +); -test.serial('GET /credential - return a new mock credential', async (t) => { - server.addCredential('b', { user: 'johnny', password: 'cash' }); +test.serial('GET /credential - return a credential', async (t) => { + server.addCredential('a', { user: 'johnny', password: 'cash' }); - const res = await get('credential/b'); + const res = await get('credential/a'); t.is(res.status, 200); const job = await res.json(); @@ -51,14 +51,6 @@ test.serial('GET /credential - return a new mock credential', async (t) => { t.is(job.password, 'cash'); }); -test.serial( - 'GET /credential - return 404 if no credential found', - async (t) => { - const res = await get('credential/c'); - t.is(res.status, 404); - } -); - test.serial( 'POST /attempts/next - return 204 and no body for an empty queue', async (t) => { @@ -75,7 +67,7 @@ test.serial('POST /attempts/next - return 400 if no id provided', async (t) => { }); test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { - server.addToQueue('attempt-1'); + server.addToQueue(attempt1); t.is(server.getQueueLength(), 1); const res = await post('attempts/next', { id: 'x' }); @@ -116,9 +108,9 @@ test.serial( ); test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { - server.addToQueue('attempt-1'); - server.addToQueue('attempt-1'); - server.addToQueue('attempt-1'); + server.addToQueue(attempt1); + server.addToQueue(attempt1); + server.addToQueue(attempt1); t.is(server.getQueueLength(), 3); const res = await post('attempts/next?count=2', { id: 'x' }); @@ -135,7 +127,7 @@ test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { test.serial( 'POST /attempts/next - clear the queue after a request', async (t) => { - server.addToQueue('attempt-1'); + server.addToQueue(attempt1); const res1 = await post('attempts/next', { id: 'x' }); t.is(res1.status, 200); @@ -194,7 +186,7 @@ test.serial( let evt; let didCall = false; - server.once('complete', (e) => { + server.once('workflow-complete', (e) => { didCall = true; evt = e; }); From 0662eb59c7ed8b20aa9bb5a78bb7af027ce65770 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 30 May 2023 17:29:43 +0100 Subject: [PATCH 015/232] rtm-server: lightning mock must receive an rtm id --- packages/rtm-server/notes | 4 + packages/rtm-server/src/mock/lightning/api.ts | 19 +++-- .../src/mock/lightning/middleware.ts | 38 ++++++--- .../rtm-server/src/mock/lightning/server.ts | 21 +++-- .../rtm-server/src/mock/runtime-manager.ts | 4 +- packages/rtm-server/src/server.ts | 8 +- packages/rtm-server/src/work-loop.ts | 2 +- packages/rtm-server/test/integration.test.ts | 5 +- .../rtm-server/test/mock/lightning.test.ts | 80 +++++++++++++------ 9 files changed, 131 insertions(+), 50 deletions(-) diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes index 66c705564..4aeb5727a 100644 --- a/packages/rtm-server/notes +++ b/packages/rtm-server/notes @@ -65,3 +65,7 @@ Should lightning and the RTM server maintain a web socket to pipe all logs throu Should logs be posted in batches? Every n-ms? I'm gonna keep it really simple for now, and send all the logs after completion in a single post. + +# JSON style + +Should the Lightning interfaces use snake case in JSON objects? Probably? They'd be closer to lightning's native style diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 667d05ab1..6b6b258c9 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -10,11 +10,21 @@ import type { ServerState } from './server'; export const API_PREFIX = `/api/1`; +interface RTMBody { + rtm_id: string; +} + +export interface FetchNextBody extends RTMBody {} + +export interface AttemptCompleteBody extends RTMBody { + state: any; // JSON state object (undefined? null?) +} + export default (router: Router, state: ServerState) => { - // Basically all requests must include an rtm id + // Basically all requests must include an rtm_id // And probably later a security token - // POST attempts/next: + // POST attempts/next // Removes Attempts from the queue and returns them to the caller // Lightning should track who has each attempt // 200 - return an array of pending attempts @@ -36,9 +46,8 @@ export default (router: Router, state: ServerState) => { // Notify an attempt has finished // Could be error or success state // If a complete comes in from an unexpected source (ie a timed out job), this should throw - // Error or state in payload - // { data } | { error } - // TODO result needs to be { rtmId, state, meta } (meta to come, but timing, memory info etc) + // state and rtm_id should be in the payload + // { rtm,_id, state } | { rtmId, error } router.post(`${API_PREFIX}/attempts/complete/:id`, createComplete(state)); // TODO i want this too: confirm that an attempt has started diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 096c8cf18..1ad4d04ff 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -1,6 +1,6 @@ import Koa from 'koa'; import type { ServerState } from './server'; -import { State } from '../../types'; +import { AttemptCompleteBody } from './api'; export const unimplemented = (ctx: Koa.Context) => { ctx.status = 501; @@ -9,13 +9,11 @@ export const unimplemented = (ctx: Koa.Context) => { export const createFetchNextJob = (state: ServerState) => (ctx: Koa.Context) => { const { queue } = state; - const { body } = ctx.request; - if (!body || !body.id) { + if (!body || !body.rtm_id) { ctx.status = 400; return; } - const countRaw = ctx.request.query.count as unknown; let count = 1; if (countRaw) { @@ -69,12 +67,30 @@ export const createNotify = (state: ServerState) => (ctx: Koa.Context) => { ctx.status = 200; }; -export const createComplete = (state: ServerState) => (ctx: Koa.Context) => { - const { results, events } = state; - const finalState = ctx.request.body as State; - results[ctx.params.id] = finalState; +export const createComplete = + (state: ServerState) => + ( + ctx: Koa.ParameterizedContext< + Koa.DefaultState, + Koa.DefaultContext, + AttemptCompleteBody + > + ) => { + const { results, events } = state; + const { state: resultState, rtm_id } = ctx.request.body; - events.emit('workflow-complete', { id: ctx.params.id, state: finalState }); + if (results[ctx.params.id] && results[ctx.params.id].rtmId === rtm_id) { + results[ctx.params.id].state = resultState; - ctx.status = 200; -}; + events.emit('workflow-complete', { + rtm_id, + workflow_id: ctx.params.id, + state: resultState, + }); + + ctx.status = 200; + } else { + // Unexpected result + ctx.status = 400; + } + }; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 6c162719d..66b17f9f1 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -17,7 +17,7 @@ export type ServerState = { credentials: Record; attempts: Record; queue: Attempt[]; - results: Record; + results: Record; events: EventEmitter; }; @@ -50,14 +50,16 @@ const createLightningServer = (options = {}) => { app.addAttempt = (attempt: Attempt) => { state.attempts[attempt.id] = attempt; }; - app.addToQueue = (attempt: string | Attempt) => { + app.addToQueue = (attempt: string | Attempt, rtmId: string = 'rtm') => { if (typeof attempt == 'string') { + app.addPendingWorkflow(attempt, rtmId); if (state.attempts[attempt]) { state.queue.push(state.attempts[attempt]); return true; } throw new Error(`attempt ${attempt} not found`); } else if (attempt) { + app.addPendingWorkflow(attempt.id, rtmId); state.queue.push(attempt); return true; } @@ -65,7 +67,7 @@ const createLightningServer = (options = {}) => { app.waitForResult = (workflowId: string) => { return new Promise((resolve) => { const handler = (evt) => { - if (evt.id === workflowId) { + if (evt.workflow_id === workflowId) { state.events.removeListener('workflow-complete', handler); resolve(evt); } @@ -73,9 +75,18 @@ const createLightningServer = (options = {}) => { state.events.addListener('workflow-complete', handler); }); }; - app.resetQueue = () => (state.queue = []); + app.addPendingWorkflow = (workflowId: string, rtmId: string) => { + state.results[workflowId] = { + rtmId, + state: null, + }; + }; + app.reset = () => { + state.queue = []; + state.results = {}; + }; app.getQueueLength = () => state.queue.length; - app.getResult = (attemptId: string) => state.results[attemptId]; + app.getResult = (attemptId: string) => state.results[attemptId]?.state; app.on = (event: 'notify', fn: (evt: any) => void) => { state.events.addListener(event, fn); }; diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 25a756cc7..d641cf51c 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -61,7 +61,7 @@ let autoServerId = 0; const assembleState = () => {}; function createMock( - serverId = autoServerId, + serverId: string, resolvers: LazyResolvers = mockResolvers ) { const activeWorkflows = {} as Record; @@ -145,7 +145,7 @@ function createMock( }; return { - id: serverId || ++autoServerId, + id: serverId || `${++autoServerId}`, on, once, execute, diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index be126157a..dd8a4e202 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -14,6 +14,7 @@ import convertAttempt from './util/convert-attempt'; import { Attempt } from './types'; const postResult = async ( + rtmId: string, lightningUrl: string, attemptId: string, state: any @@ -23,7 +24,10 @@ const postResult = async ( `${lightningUrl}/api/1/attempts/complete/${attemptId}`, { method: 'POST', - body: JSON.stringify(state || {}), + body: JSON.stringify({ + rtm_id: rtmId, + state: state, + }), headers: { Accept: 'application/json', 'Content-Type': 'application/json', @@ -67,7 +71,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { // TODO how about an 'all' so we can "route" events? rtm.on('workflow-complete', ({ id, state }) => { - postResult(options.lightning!, id, state); + postResult(rtm.id, options.lightning!, id, state); }); // TMP doing this for tests but maybe its better done externally diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index 3f1f30e68..7953a3c1a 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -12,7 +12,7 @@ export default ( // TODO what if this retuns like a 500? Server down? const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { method: 'POST', - body: JSON.stringify({ id: rtmId }), + body: JSON.stringify({ rtm_id: rtmId }), headers: { Accept: 'application/json', 'Content-Type': 'application/json', diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 857b42840..c2266106f 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -19,7 +19,10 @@ const urls = { test.before(() => { lng = createLightningServer({ port: 7654 }); - rtm = createRTMServer(createMockRTM(), { port: 4567, lightning: urls.lng }); + rtm = createRTMServer(createMockRTM('rtm'), { + port: 4567, + lightning: urls.lng, + }); }); // Really high level test diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 07c56fd04..c773c86f9 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -11,7 +11,7 @@ test.before(() => { }); test.afterEach(() => { - server.resetQueue(); + server.reset(); }); test.after(() => { @@ -55,7 +55,7 @@ test.serial( 'POST /attempts/next - return 204 and no body for an empty queue', async (t) => { t.is(server.getQueueLength(), 0); - const res = await post('attempts/next', { id: 'x' }); + const res = await post('attempts/next', { rtm_id: 'rtm' }); t.is(res.status, 204); t.false(res.bodyUsed); } @@ -70,7 +70,7 @@ test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { server.addToQueue(attempt1); t.is(server.getQueueLength(), 1); - const res = await post('attempts/next', { id: 'x' }); + const res = await post('attempts/next', { rtm_id: 'rtm' }); const result = await res.json(); t.is(res.status, 200); @@ -92,7 +92,7 @@ test.serial( server.addToQueue({ id: 'abc' }); t.is(server.getQueueLength(), 1); - const res = await post('attempts/next', { id: 'x' }); + const res = await post('attempts/next', { rtm_id: 'rtm' }); t.is(res.status, 200); const result = await res.json(); @@ -113,7 +113,7 @@ test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { server.addToQueue(attempt1); t.is(server.getQueueLength(), 3); - const res = await post('attempts/next?count=2', { id: 'x' }); + const res = await post('attempts/next?count=2', { rtm_id: 'rtm' }); t.is(res.status, 200); const result = await res.json(); @@ -128,12 +128,12 @@ test.serial( 'POST /attempts/next - clear the queue after a request', async (t) => { server.addToQueue(attempt1); - const res1 = await post('attempts/next', { id: 'x' }); + const res1 = await post('attempts/next', { rtm_id: 'rtm' }); t.is(res1.status, 200); const result1 = await res1.json(); t.is(result1.length, 1); - const res2 = await post('attempts/next', { id: 'x' }); + const res2 = await post('attempts/next', { rtm_id: 'rtm' }); t.is(res2.status, 204); t.falsy(res2.bodyUsed); } @@ -172,37 +172,71 @@ test.serial( ); test.serial('POST /attempts/complete - return final state', async (t) => { + server.addPendingWorkflow('a', 'rtm'); const { status } = await post('attempts/complete/a', { - x: 10, + rtm_id: 'rtm', + state: { + x: 10, + }, }); t.is(status, 200); const result = server.getResult('a'); t.deepEqual(result, { x: 10 }); }); +test.serial('POST /attempts/complete - reject if unknown rtm', async (t) => { + const { status } = await post('attempts/complete/a', { + rtm_id: 'rtm', + state: { + x: 10, + }, + }); + t.is(status, 400); + t.falsy(server.getResult('a')); +}); + test.serial( - 'POST /attempts/complete - should echo to event emitter', + 'POST /attempts/complete - reject if unknown workflow', async (t) => { - let evt; - let didCall = false; - - server.once('workflow-complete', (e) => { - didCall = true; - evt = e; - }); + server.addPendingWorkflow('b', 'rtm'); const { status } = await post('attempts/complete/a', { - data: { - answer: 42, + rtm_id: 'rtm', + state: { + x: 10, }, }); - t.is(status, 200); - t.true(didCall); - t.truthy(evt); - t.is(evt.id, 'a'); - t.deepEqual(evt.state, { data: { answer: 42 } }); + t.is(status, 400); + t.falsy(server.getResult('a')); } ); +test.serial('POST /attempts/complete - echo to event emitter', async (t) => { + server.addPendingWorkflow('a', 'rtm'); + let evt; + let didCall = false; + + server.once('workflow-complete', (e) => { + didCall = true; + evt = e; + }); + + const { status } = await post('attempts/complete/a', { + rtm_id: 'rtm', + state: { + data: { + answer: 42, + }, + }, + }); + t.is(status, 200); + t.true(didCall); + + t.truthy(evt); + t.is(evt.rtm_id, 'rtm'); + t.is(evt.workflow_id, 'a'); + t.deepEqual(evt.state, { data: { answer: 42 } }); +}); + // test lightning should get the finished state through a helper API From f37959f2179c0c4497b94f3ddf462e5bd11bc986 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 31 May 2023 12:53:27 +0100 Subject: [PATCH 016/232] rtm-server: add logging support --- packages/rtm-server/notes | 16 ++++++++ packages/rtm-server/src/mock/lightning/api.ts | 11 +++--- .../src/mock/lightning/middleware.ts | 37 +++++++++++++------ .../rtm-server/src/mock/runtime-manager.ts | 20 ++++------ packages/rtm-server/src/server.ts | 24 ++++++++++++ .../rtm-server/test/mock/lightning.test.ts | 33 ++++++++++++----- .../test/mock/runtime-manager.test.ts | 14 +++++-- 7 files changed, 113 insertions(+), 42 deletions(-) diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes index 4aeb5727a..ffcb901eb 100644 --- a/packages/rtm-server/notes +++ b/packages/rtm-server/notes @@ -66,6 +66,22 @@ Should logs be posted in batches? Every n-ms? I'm gonna keep it really simple for now, and send all the logs after completion in a single post. +For now, we're gonna keep it simple and post all the logs right away. But pass an array for future proofing. + +But there are different types of logs! + +There's system logs and reporting. Stuff that the runtime is doing. + +And there's job logs - things which cem right out of the job vm (including the adaptor) + +The logs from the RTM are different to the CLI logs, but there are similarites. Like the CLI will print out all the versions. How will the RTM do this? What do we actually want to log? And how does it feedback? + +Do we want `log-rtm` and `log-job` ? + +What about compiler logs? The compiler functionality is gonna be a bit different, espeicially when usng the cache. + +Ok, again, I'm gonna keep it simple. Just do log logging. + # JSON style Should the Lightning interfaces use snake case in JSON objects? Probably? They'd be closer to lightning's native style diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 6b6b258c9..e158394f4 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -3,7 +3,7 @@ import { unimplemented, createFetchNextJob, createGetCredential, - createNotify, + createLog, createComplete, } from './middleware'; import type { ServerState } from './server'; @@ -37,11 +37,10 @@ export default (router: Router, state: ServerState) => { // 404 - credential not found router.get(`${API_PREFIX}/credential/:id`, createGetCredential(state)); - // Notify of some job update - // proxy to event emitter - // { event: 'event-name', ...data } - // TODO this should use a websocket to handle the high volume of logs - router.post(`${API_PREFIX}/attempts/notify/:id`, createNotify(state)); + // Notify for a batch of job logs + // [{ rtm_id, logs: ['hello world' ] }] + // TODO this could use a websocket to handle the high volume of logs + router.post(`${API_PREFIX}/attempts/log/:id`, createLog(state)); // Notify an attempt has finished // Could be error or success state diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 1ad4d04ff..9ce699a17 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -2,6 +2,19 @@ import Koa from 'koa'; import type { ServerState } from './server'; import { AttemptCompleteBody } from './api'; +// basically an author handler +const shouldAcceptRequest = ( + state: ServerState, + jobId: string, + request: Koa.Request +) => { + const { results } = state; + if (request.body) { + const { rtm_id } = request.body; + return results[jobId] && results[jobId].rtmId === rtm_id; + } +}; + export const unimplemented = (ctx: Koa.Context) => { ctx.status = 501; }; @@ -52,19 +65,21 @@ export const createGetCredential = } }; -export const createNotify = (state: ServerState) => (ctx: Koa.Context) => { +export const createLog = (state: ServerState) => (ctx: Koa.Context) => { const { events } = state; - const { event: name, ...payload } = ctx.request.body; - - const event = { - id: ctx.params.id, - name, - ...payload, // spread payload? - }; + if (shouldAcceptRequest(state, ctx.params.id, ctx.request)) { + const data = ctx.request.body; + const event = { + id: ctx.params.id, + logs: data.logs, + }; - events.emit('notify', event); + events.emit('log', event); - ctx.status = 200; + ctx.status = 200; + } else { + ctx.status = 400; + } }; export const createComplete = @@ -79,7 +94,7 @@ export const createComplete = const { results, events } = state; const { state: resultState, rtm_id } = ctx.request.body; - if (results[ctx.params.id] && results[ctx.params.id].rtmId === rtm_id) { + if (shouldAcceptRequest(state, ctx.params.id, ctx.request)) { results[ctx.params.id].state = resultState; events.emit('workflow-complete', { diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index d641cf51c..fbc9daaf2 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -14,16 +14,15 @@ type Resolver = (id: string) => Promise; // A list of helper functions which basically resolve ids into JSON // to lazy load assets export type LazyResolvers = { - credentials: Resolver; - state: Resolver; - expressions: Resolver; + credentials?: Resolver; + state?: Resolver; + expressions?: Resolver; }; export type RTMEvent = | 'job-start' | 'job-complete' - | 'job-log' - //| 'job-error' + | 'log' // this is a log from inside the VM | 'workflow-start' // before compile | 'workflow-complete' // after everything has run | 'workflow-error'; // ? @@ -61,7 +60,7 @@ let autoServerId = 0; const assembleState = () => {}; function createMock( - serverId: string, + serverId?: string, resolvers: LazyResolvers = mockResolvers ) { const activeWorkflows = {} as Record; @@ -97,16 +96,13 @@ function createMock( // start instantly and emit as it goes dispatch('job-start', { id, runId }); - // TODO random timeout - // What is a job log? Anything emitted by the RTM I guess? - // Namespaced to compile, r/t, job etc. - // It's the json output of the logger - dispatch('job-log', { id, runId }); - let state = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { state = JSON.parse(expression); + // What does this look like? Should be a logger object + dispatch('log', { message: ['Parsing expression as JSON state'] }); + dispatch('log', { message: [state] }); } catch (e) { // Do nothing, it's fine } diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index dd8a4e202..50759e594 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -39,6 +39,26 @@ const postResult = async ( // Backoff and try again? }; +// Send a batch of logs +const postLog = async ( + rtmId: string, + lightningUrl: string, + attemptId: string, + messages: any[] +) => { + await fetch(`${lightningUrl}/api/1/attempts/log/${attemptId}`, { + method: 'POST', + body: JSON.stringify({ + rtm_id: rtmId, + logs: messages, + }), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); +}; + type ServerOptions = { backoff?: number; maxWorkflows?: number; @@ -74,6 +94,10 @@ function createServer(rtm: any, options: ServerOptions = {}) { postResult(rtm.id, options.lightning!, id, state); }); + rtm.on('log', ({ id, messages }) => { + postLog(rtm.id, options.lightning!, id, messages); + }); + // TMP doing this for tests but maybe its better done externally app.on = (...args) => rtm.on(...args); app.once = (...args) => rtm.once(...args); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index c773c86f9..05c25a139 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -139,35 +139,48 @@ test.serial( } ); -// TODO this API is gonna be restructured -test.serial('POST /attempts/notify - should return 200', async (t) => { - const { status } = await post('attempts/notify/a', {}); +test.serial('POST /attempts/log - should return 200', async (t) => { + server.addPendingWorkflow('a', 'rtm'); + const { status } = await post('attempts/log/a', { + rtm_id: 'rtm', + logs: [{ message: 'hello world' }], + }); t.is(status, 200); }); +test.serial( + 'POST /attempts/log - should return 400 if no rtm_id', + async (t) => { + const { status } = await post('attempts/log/a', { + rtm_id: 'rtm', + logs: [{ message: 'hello world' }], + }); + t.is(status, 400); + } +); + test.serial( 'POST /attempts/notify - should echo to event emitter', async (t) => { + server.addPendingWorkflow('a', 'rtm'); let evt; let didCall = false; - server.once('notify', (e) => { + server.once('log', (e) => { didCall = true; evt = e; }); - const { status } = await post('attempts/notify/a', { - event: 'job-start', - count: 101, + const { status } = await post('attempts/log/a', { + rtm_id: 'rtm', + logs: [{ message: 'hello world' }], }); t.is(status, 200); t.true(didCall); - // await wait(() => evt); t.truthy(evt); t.is(evt.id, 'a'); - t.is(evt.name, 'job-start'); - t.is(evt.count, 101); + t.deepEqual(evt.logs, [{ message: 'hello world' }]); } ); diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index 761119a32..a0745103f 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -20,9 +20,9 @@ const sampleWorkflow = { } as ExecutionPlan; test('mock runtime manager should have an id', (t) => { - const rtm = create(22); + const rtm = create('22'); const keys = Object.keys(rtm); - t.assert(rtm.id == 22); + t.assert(rtm.id == '22'); // No need to test the full API, just make sure it smells right t.assert(keys.includes('on')); @@ -100,6 +100,14 @@ test('mock should evaluate expressions as JSON', async (t) => { t.deepEqual(evt.state, { x: 10 }); }); +test('mock should dispatch log events when evaluating JSON', async (t) => { + const rtm = create(); + + rtm.execute(sampleWorkflow); + const evt = await waitForEvent(rtm, 'log'); + t.deepEqual(evt.message, ['Parsing expression as JSON state']); +}); + test('resolve credential before job-start if credential is a string', async (t) => { const wf = clone(sampleWorkflow); wf.jobs[0].configuration = 'x'; @@ -110,7 +118,7 @@ test('resolve credential before job-start if credential is a string', async (t) return {}; }; - const rtm = create(1, { credentials }); + const rtm = create('1', { credentials }); rtm.execute(wf); await waitForEvent(rtm, 'job-start'); From 19b23c07d8d4d61513857f14dabe2723a1f0511b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 31 May 2023 15:33:23 +0100 Subject: [PATCH 017/232] rtm-server: allow server to start from dev console with post api --- packages/rtm-server/package.json | 7 +++-- packages/rtm-server/src/api.ts | 10 ++++-- .../rtm-server/src/middleware/workflow.ts | 12 +++++++ packages/rtm-server/src/server.ts | 25 +++++++++++++-- packages/rtm-server/src/start.ts | 31 +++++++++++++++++++ packages/rtm-server/test/integration.test.ts | 2 +- packages/rtm-server/tsconfig.json | 5 ++- pnpm-lock.yaml | 6 ++++ 8 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 packages/rtm-server/src/middleware/workflow.ts create mode 100644 packages/rtm-server/src/start.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index b332ecf10..d0b481f3f 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -10,15 +10,18 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", - "start": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/index.ts'" + "start": "tsm --experimental-vm-modules --no-warnings src/start.ts", + "_start": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/index.ts'" }, "author": "Open Function Group ", "license": "ISC", "dependencies": { "@koa/router": "^12.0.0", + "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", "koa": "^2.13.4", - "koa-bodyparser": "^4.4.0" + "koa-bodyparser": "^4.4.0", + "yargs": "^17.5.1" }, "devDependencies": { "@types/koa": "^2.13.5", diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts index 514392a01..dc1f2ff3d 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api.ts @@ -1,17 +1,23 @@ import Router from '@koa/router'; import healthcheck from './middleware/healthcheck'; +import workflow from './middleware/workflow'; + // this defines the main API routes in a nice central place // So what is the API of this server? // It's mostly a pull model, apart from I think the healthcheck -// So it doesn't need much +// Should have diagnostic and reporting APIs +// maybe even a simple frontend? -const createAPI = () => { +const createAPI = (logger, execute) => { const router = new Router(); router.get('/healthcheck', healthcheck); + // Dev API to run a workflow + router.post('/workflow', workflow(execute, logger)); + return router; }; diff --git a/packages/rtm-server/src/middleware/workflow.ts b/packages/rtm-server/src/middleware/workflow.ts new file mode 100644 index 000000000..01a243045 --- /dev/null +++ b/packages/rtm-server/src/middleware/workflow.ts @@ -0,0 +1,12 @@ +export default (execute, logger) => (ctx) => { + logger.info('POST TO WORKFLOW'); + try { + const attempt = ctx.request.body; + // TODO should this return the result... ? + // No other way to get hold of it + execute(attempt); + ctx.status = 200; + } catch (e: any) { + logger.error('Error starting attempt'); + } +}; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 50759e594..086536fa9 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -8,10 +8,13 @@ */ import Koa from 'koa'; +import bodyParser from 'koa-bodyparser'; + import createAPI from './api'; import startWorkLoop from './work-loop'; import convertAttempt from './util/convert-attempt'; import { Attempt } from './types'; +// import createLogger, { createMockLogger, Logger } from '@openfn/logger'; const postResult = async ( rtmId: string, @@ -65,36 +68,54 @@ type ServerOptions = { port?: number; lightning?: string; // url to lightning instance rtm?: any; + logger?: Logger; }; function createServer(rtm: any, options: ServerOptions = {}) { + // const logger = options.logger || createMockLogger(); + const logger = console; + const port = options.port || 1234; + + logger.info('Starting server'); const app = new Koa(); + app.use(bodyParser()); + const execute = (attempt: Attempt) => { const plan = convertAttempt(attempt); rtm.execute(plan); }; - const apiRouter = createAPI(); + // TODO actually it's a bit silly to pass everything through, why not just declare the route here? + // Or maybe I need a central controller/state object + const apiRouter = createAPI(logger, execute); app.use(apiRouter.routes()); app.use(apiRouter.allowedMethods()); - app.listen(options.port || 1234); + app.listen(port); + logger.info('Listening on', port); app.destroy = () => { // TODO close the work loop + logger.info('Closing server'); }; if (options.lightning) { + logger.log('Starting work loop at', options.lightning); startWorkLoop(options.lightning, rtm.id, execute); + } else { + logger.warn('No lightning URL provided'); } // TODO how about an 'all' so we can "route" events? rtm.on('workflow-complete', ({ id, state }) => { + logger.log(`${id}: workflow complete: `, id); + logger.log(state); postResult(rtm.id, options.lightning!, id, state); }); rtm.on('log', ({ id, messages }) => { + logger.log(`${id}: `, ...messages); postLog(rtm.id, options.lightning!, id, messages); }); diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts new file mode 100644 index 000000000..cab14ebe3 --- /dev/null +++ b/packages/rtm-server/src/start.ts @@ -0,0 +1,31 @@ +// start the server in a local CLI +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import createLogger from '@openfn/logger'; + +import createMockRTM from './mock/runtime-manager'; +import createRTMServer from './server'; + +const args = yargs(hideBin(process.argv)) + .command('server', 'Start a runtime manager server') + .option('port', { + alias: 'p', + description: 'Port to run the server on', + default: 2222, + }) + .option('lightning', { + alias: 'l', + description: 'Base url to Lightning', + }) + .parse(); + +const rtm = createMockRTM('rtm'); + +// TODO why is this blowing up?? +// const logger = createLogger('SRV'); + +createRTMServer(rtm, { + port: args.port, + lightning: args.lightning, + // logger, +}); diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index c2266106f..6a23f4ddf 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -41,7 +41,7 @@ test.serial('process an attempt', async (t) => { t.is(state.answer, 42); }); -// process multple attempts +// process multiple attempts test.serial.skip( 'should post to attempts/complete with the final state', diff --git a/packages/rtm-server/tsconfig.json b/packages/rtm-server/tsconfig.json index b3d766fc1..ba1452256 100644 --- a/packages/rtm-server/tsconfig.json +++ b/packages/rtm-server/tsconfig.json @@ -1,4 +1,7 @@ { "extends": "../../tsconfig.common", - "include": ["src/**/*.ts"] + "include": ["src/**/*.ts"], + "compilerOptions": { + "module": "ESNext" + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e5772518..943fa6902 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -329,6 +329,9 @@ importers: '@koa/router': specifier: ^12.0.0 version: 12.0.0 + '@openfn/logger': + specifier: workspace:* + version: link:../logger '@openfn/runtime': specifier: workspace:* version: link:../runtime @@ -338,6 +341,9 @@ importers: koa-bodyparser: specifier: ^4.4.0 version: 4.4.0 + yargs: + specifier: ^17.5.1 + version: 17.7.2 devDependencies: '@types/koa': specifier: ^2.13.5 From b4e918a88a7f6061c4e0918aecb792b78cab53b4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 31 May 2023 18:36:15 +0100 Subject: [PATCH 018/232] runtime-manager: start moving API over to new style Currently broken when executing --- packages/rtm-server/package.json | 13 +- packages/rtm-server/src/api.ts | 4 +- .../rtm-server/src/middleware/healthcheck.ts | 2 +- .../rtm-server/src/middleware/workflow.ts | 3 +- packages/rtm-server/src/server.ts | 37 ++-- packages/rtm-server/src/start.ts | 18 +- .../rtm-server/src/util/convert-attempt.ts | 2 + .../rtm-server/src/util/try-with-backoff.ts | 2 +- packages/runtime-manager/package.json | 8 +- packages/runtime-manager/src/Manager.ts | 123 ------------ packages/runtime-manager/src/events.ts | 6 +- packages/runtime-manager/src/index.ts | 6 +- packages/runtime-manager/src/rtm.ts | 181 ++++++++++++++++++ .../src/runners/autoinstall.ts | 10 + .../runtime-manager/src/runners/compile.ts | 63 ++++++ .../runtime-manager/src/runners/execute.ts | 31 +++ packages/runtime-manager/src/server/index.ts | 2 +- packages/runtime-manager/src/worker-helper.ts | 4 +- packages/runtime-manager/src/worker.ts | 14 +- packages/runtime-manager/test/manager.test.ts | 2 +- pnpm-lock.yaml | 33 +++- 21 files changed, 385 insertions(+), 179 deletions(-) delete mode 100644 packages/runtime-manager/src/Manager.ts create mode 100644 packages/runtime-manager/src/rtm.ts create mode 100644 packages/runtime-manager/src/runners/autoinstall.ts create mode 100644 packages/runtime-manager/src/runners/compile.ts create mode 100644 packages/runtime-manager/src/runners/execute.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index d0b481f3f..32ffee358 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -10,7 +10,7 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", - "start": "tsm --experimental-vm-modules --no-warnings src/start.ts", + "start": "ts-node-esm src/start.ts", "_start": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/index.ts'" }, "author": "Open Function Group ", @@ -19,19 +19,22 @@ "@koa/router": "^12.0.0", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", + "@openfn/runtime-manager": "workspace:*", "koa": "^2.13.4", - "koa-bodyparser": "^4.4.0", - "yargs": "^17.5.1" + "koa-bodyparser": "^4.4.0" }, "devDependencies": { "@types/koa": "^2.13.5", + "@types/koa-bodyparser": "^4.3.10", + "@types/koa-router": "^7.4.4", "@types/node": "^18.15.3", + "@types/yargs": "^17.0.12", "ava": "5.1.0", "nodemon": "^2.0.19", "ts-node": "^10.9.1", "tslib": "^2.4.0", - "tsm": "^2.2.2", "tsup": "^6.2.3", - "typescript": "^4.6.4" + "typescript": "^4.6.4", + "yargs": "^17.6.2" } } diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts index dc1f2ff3d..316adc1a3 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api.ts @@ -1,4 +1,6 @@ +// @ts-ignore import Router from '@koa/router'; +import type { Logger } from '@openfn/logger'; import healthcheck from './middleware/healthcheck'; import workflow from './middleware/workflow'; @@ -10,7 +12,7 @@ import workflow from './middleware/workflow'; // Should have diagnostic and reporting APIs // maybe even a simple frontend? -const createAPI = (logger, execute) => { +const createAPI = (logger: Logger, execute: any) => { const router = new Router(); router.get('/healthcheck', healthcheck); diff --git a/packages/rtm-server/src/middleware/healthcheck.ts b/packages/rtm-server/src/middleware/healthcheck.ts index 658dc7564..10ddafd76 100644 --- a/packages/rtm-server/src/middleware/healthcheck.ts +++ b/packages/rtm-server/src/middleware/healthcheck.ts @@ -1,4 +1,4 @@ -export default (ctx) => { +export default (ctx: any) => { ctx.status = 200; ctx.body = 'OK'; }; diff --git a/packages/rtm-server/src/middleware/workflow.ts b/packages/rtm-server/src/middleware/workflow.ts index 01a243045..78f35f23e 100644 --- a/packages/rtm-server/src/middleware/workflow.ts +++ b/packages/rtm-server/src/middleware/workflow.ts @@ -1,4 +1,4 @@ -export default (execute, logger) => (ctx) => { +export default (execute: any, logger: any) => (ctx: any) => { logger.info('POST TO WORKFLOW'); try { const attempt = ctx.request.body; @@ -8,5 +8,6 @@ export default (execute, logger) => (ctx) => { ctx.status = 200; } catch (e: any) { logger.error('Error starting attempt'); + console.log(e); } }; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 086536fa9..813987fdd 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -14,7 +14,7 @@ import createAPI from './api'; import startWorkLoop from './work-loop'; import convertAttempt from './util/convert-attempt'; import { Attempt } from './types'; -// import createLogger, { createMockLogger, Logger } from '@openfn/logger'; +import { createMockLogger, Logger } from '@openfn/logger'; const postResult = async ( rtmId: string, @@ -23,20 +23,17 @@ const postResult = async ( state: any ) => { if (lightningUrl) { - const result = await fetch( - `${lightningUrl}/api/1/attempts/complete/${attemptId}`, - { - method: 'POST', - body: JSON.stringify({ - rtm_id: rtmId, - state: state, - }), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - } - ); + await fetch(`${lightningUrl}/api/1/attempts/complete/${attemptId}`, { + method: 'POST', + body: JSON.stringify({ + rtm_id: rtmId, + state: state, + }), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); } // TODO what if result is not 200? // Backoff and try again? @@ -72,8 +69,7 @@ type ServerOptions = { }; function createServer(rtm: any, options: ServerOptions = {}) { - // const logger = options.logger || createMockLogger(); - const logger = console; + const logger = options.logger || createMockLogger(); const port = options.port || 1234; logger.info('Starting server'); @@ -95,7 +91,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { app.listen(port); logger.info('Listening on', port); - app.destroy = () => { + (app as any).destroy = () => { // TODO close the work loop logger.info('Closing server'); }; @@ -108,13 +104,13 @@ function createServer(rtm: any, options: ServerOptions = {}) { } // TODO how about an 'all' so we can "route" events? - rtm.on('workflow-complete', ({ id, state }) => { + rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { logger.log(`${id}: workflow complete: `, id); logger.log(state); postResult(rtm.id, options.lightning!, id, state); }); - rtm.on('log', ({ id, messages }) => { + rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { logger.log(`${id}: `, ...messages); postLog(rtm.id, options.lightning!, id, messages); }); @@ -126,6 +122,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { // debug API to run a workflow // Used in unit tests // Only loads in dev mode? + // @ts-ignore app.execute = execute; return app; diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index cab14ebe3..9dcc18518 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -3,29 +3,37 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import createLogger from '@openfn/logger'; -import createMockRTM from './mock/runtime-manager'; +import createRTM from '@openfn/runtime-manager'; import createRTMServer from './server'; +type Args = { + _: string[]; + port?: number; + lightning?: string; +}; + const args = yargs(hideBin(process.argv)) .command('server', 'Start a runtime manager server') .option('port', { alias: 'p', description: 'Port to run the server on', + type: 'number', default: 2222, }) .option('lightning', { alias: 'l', description: 'Base url to Lightning', }) - .parse(); + .parse() as Args; -const rtm = createMockRTM('rtm'); +const rtm = createRTM(); +console.log('RTM created'); // TODO why is this blowing up?? -// const logger = createLogger('SRV'); +const logger = createLogger('SRV'); createRTMServer(rtm, { port: args.port, lightning: args.lightning, - // logger, + logger, }); diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/rtm-server/src/util/convert-attempt.ts index be86b2af3..1ca9f7664 100644 --- a/packages/rtm-server/src/util/convert-attempt.ts +++ b/packages/rtm-server/src/util/convert-attempt.ts @@ -23,6 +23,7 @@ export default (attempt: Attempt): ExecutionPlan => { const connectedEdges = edges.filter((e) => e.source_trigger_id === id); if (connectedEdges.length) { nodes[id].next = connectedEdges.reduce((obj, edge) => { + // @ts-ignore obj[edge.target_job_id] = true; return obj; }, {}); @@ -46,6 +47,7 @@ export default (attempt: Attempt): ExecutionPlan => { const next = edges .filter((e) => e.source_job_id === id) .reduce((obj, edge) => { + // @ts-ignore obj[edge.target_job_id] = edge.condition ? { expression: edge.condition } : true; diff --git a/packages/rtm-server/src/util/try-with-backoff.ts b/packages/rtm-server/src/util/try-with-backoff.ts index 7170a7e99..a3c923fab 100644 --- a/packages/rtm-server/src/util/try-with-backoff.ts +++ b/packages/rtm-server/src/util/try-with-backoff.ts @@ -15,7 +15,7 @@ type Options = { // under what circumstance should this function throw? // If it timesout // Can the inner function force a throw? An exit early? -const tryWithBackoff = (fn, opts: Options = {}) => { +const tryWithBackoff = (fn: any, opts: Options = {}) => { if (!opts.timeout) { opts.timeout = 100; // TODO errors occur if this is too low? } diff --git a/packages/runtime-manager/package.json b/packages/runtime-manager/package.json index fdc53e731..ed61f69bf 100644 --- a/packages/runtime-manager/package.json +++ b/packages/runtime-manager/package.json @@ -2,7 +2,7 @@ "name": "@openfn/runtime-manager", "version": "0.0.37", "description": "An example runtime manager service.", - "main": "index.js", + "main": "dist/index.js", "type": "module", "private": true, "scripts": { @@ -35,5 +35,9 @@ "tsm": "^2.2.2", "tsup": "^6.2.3", "typescript": "^4.6.4" - } + }, + "files": [ + "dist", + "README.md" + ] } diff --git a/packages/runtime-manager/src/Manager.ts b/packages/runtime-manager/src/Manager.ts deleted file mode 100644 index 646e99b02..000000000 --- a/packages/runtime-manager/src/Manager.ts +++ /dev/null @@ -1,123 +0,0 @@ -import path from 'node:path'; -import workerpool from 'workerpool'; -import * as e from './events'; -import compile from '@openfn/compiler'; - -export type State = any; // TODO I want a nice state def with generics - -// hmm, may need to support this for unit tests (which does kind of make sense) -type LiveJob = Array<(s: State) => State>; - -type JobRegistry = Record; - -let jobid = 1000; - -// Archive of every job we've run -// Fien to just keep in memory for now -type JobStats = { - id: number; - name: string; - status: 'pending' | 'done' | 'err'; - startTime: number; - threadId: number; - duration: number; - error?: string; - result?: any; // State -}; - -// Manages a pool of workers -const Manager = function (useMock = false) { - const jobsList: Map = new Map(); - const activeJobs: number[] = []; - - const registry: JobRegistry = {}; - const workers = workerpool.pool( - path.resolve(useMock ? './dist/mock-worker.js' : './dist/worker.js') - ); - - const acceptJob = (jobId: number, name: string, threadId: number) => { - if (jobsList.has(jobId)) { - throw new Error(`Job with id ${jobId} is already defined`); - } - jobsList.set(jobId, { - id: jobId, - name, - status: 'pending', - threadId, - startTime: new Date().getTime(), - duration: -1, - }); - activeJobs.push(jobId); - }; - - const completeJob = (jobId: number, state: any) => { - if (!jobsList.has(jobId)) { - throw new Error(`Job with id ${jobId} is not defined`); - } - const job = jobsList.get(jobId)!; - job.status = 'done'; - job.result = state; - job.duration = new Date().getTime() - job.startTime; - const idx = activeJobs.findIndex((id) => id === jobId); - activeJobs.splice(idx, 1); - }; - - // Run a job in a worker - // Accepts the name of a registered job - const run = async (name: string, state?: any): Promise => { - const src = registry[name]; - if (src) { - const thisJobId = ++jobid; - - await workers.exec('run', [jobid, src, state], { - on: ({ type, ...args }: e.JobEvent) => { - if (type === e.ACCEPT_JOB) { - const { jobId, threadId } = args as e.AcceptJobEvent; - acceptJob(jobId, name, threadId); - } else if (type === e.COMPLETE_JOB) { - const { jobId, state } = args as e.CompleteJobEvent; - completeJob(jobId, state); - } - }, - }); - return jobsList.get(thisJobId) as JobStats; - } - throw new Error('Job not found: ' + name); - }; - - // register a job to enable it to be run - // The job will be compiled - const registerJob = (name: string, source: string) => { - if (registry[name]) { - throw new Error('Job already registered: ' + name); - } - registry[name] = compile(source); - }; - - const getRegisteredJobs = () => Object.keys(registry); - - const getActiveJobs = (): JobStats[] => { - const jobs = activeJobs.map((id) => jobsList.get(id)); - return jobs.filter((j) => j) as JobStats[]; // no-op for typings - }; - - const getCompletedJobs = (): JobStats[] => { - return Array.from(jobsList.values()).filter((job) => job.status === 'done'); - }; - - const getErroredJobs = (): JobStats[] => { - return Array.from(jobsList.values()).filter((job) => job.status === 'err'); - }; - - return { - _registry: registry, // for unit testing really - run, - registerJob, - getRegisteredJobs, - getActiveJobs, - getCompletedJobs, - getErroredJobs, - }; -}; - -export default Manager; diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index b92461f3e..34c8f2923 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -8,19 +8,19 @@ type State = any; // TODO export type AcceptJobEvent = { type: typeof ACCEPT_JOB; - jobId: number; + jobId: string; threadId: number; }; export type CompleteJobEvent = { type: typeof COMPLETE_JOB; - jobId: number; + jobId: string; state: State; }; export type ErrJobEvent = { type: typeof JOB_ERROR; - jobId: number; + jobId: string; message: string; }; diff --git a/packages/runtime-manager/src/index.ts b/packages/runtime-manager/src/index.ts index 15c95fb2a..db8e8169e 100644 --- a/packages/runtime-manager/src/index.ts +++ b/packages/runtime-manager/src/index.ts @@ -1,5 +1,3 @@ -import Manager from './Manager'; +import createRTM from './rtm'; -// Not interested in exporting the sever stuff here, just the acutal runtime service - -export default Manager; +export default createRTM; diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts new file mode 100644 index 000000000..1c3a6a868 --- /dev/null +++ b/packages/runtime-manager/src/rtm.ts @@ -0,0 +1,181 @@ +import path from 'node:path'; +import crypto from 'node:crypto'; +import { EventEmitter } from 'node:events'; +import workerpool from 'workerpool'; +import { ExecutionPlan } from '@openfn/runtime'; + +import * as e from './events'; +import createAutoinstall from './runners/autoinstall'; +import createCompile from './runners/compile'; +import createExecute from './runners/execute'; + +export type State = any; // TODO I want a nice state def with generics + +// hmm, may need to support this for unit tests (which does kind of make sense) +type LiveJob = Array<(s: State) => State>; + +type JobRegistry = Record; + +let jobid = 1000; + +// Archive of every job we've run +// Fien to just keep in memory for now +type JobStats = { + id: number; + name: string; + status: 'pending' | 'done' | 'err'; + startTime: number; + threadId: number; + duration: number; + error?: string; + result?: any; // State +}; + +type Resolver = (id: string) => Promise; + +// A list of helper functions which basically resolve ids into JSON +// to lazy load assets +export type LazyResolvers = { + credentials?: Resolver; + state?: Resolver; + expressions?: Resolver; +}; + +const createRTM = function ( + serverId?: string, + resolvers?: LazyResolvers, + useMock = false +) { + const id = serverId || crypto.randomUUID(); + + const jobsList: Map = new Map(); + const activeJobs: number[] = []; + + const registry: JobRegistry = {}; + const workers = workerpool.pool( + path.resolve(useMock ? './dist/mock-worker.js' : './dist/worker.js') + ); + + const events = new EventEmitter(); + + const acceptJob = (jobId: number, name: string, threadId: number) => { + console.log('>> Accept job'); + if (jobsList.has(jobId)) { + throw new Error(`Job with id ${jobId} is already defined`); + } + jobsList.set(jobId, { + id: jobId, + name, + status: 'pending', + threadId, + startTime: new Date().getTime(), + duration: -1, + }); + activeJobs.push(jobId); + }; + + const completeJob = (jobId: number, state: any) => { + console.log('>> complete job'); + if (!jobsList.has(jobId)) { + throw new Error(`Job with id ${jobId} is not defined`); + } + const job = jobsList.get(jobId)!; + job.status = 'done'; + job.result = state; + job.duration = new Date().getTime() - job.startTime; + const idx = activeJobs.findIndex((id) => id === jobId); + activeJobs.splice(idx, 1); + }; + + const execute = createExecute(workers, acceptJob); + const compile = createCompile(console as any, '/tmp/openfn/repo'); + + // How much of this happens inside the worker? + // Shoud the main thread handle compilation? Has to if we want to cache + // Unless we create a dedicated compiler worker + // TODO error handling, timeout + const handleExecute = async (plan: ExecutionPlan) => { + // autoinstall + // compile it + const compiledPlan = await compile(plan); + console.log(JSON.stringify(compiledPlan, null, 2)); + + const result = await execute(compiledPlan); + console.log('RESULT', result); + completeJob(plan.id!, result); + + // Return the result + // Note that the mock doesn't behave like ths + // And tbf I don't think we should keep the promise open - there's no point? + return result; + }; + + // // Run a job in a worker + // // Accepts the name of a registered job + // const run = async (name: string, state?: any): Promise => { + // const src = registry[name]; + // if (src) { + // const thisJobId = ++jobid; + + // await workers.exec('run', [jobid, src, state], { + // on: ({ type, ...args }: e.JobEvent) => { + // if (type === e.ACCEPT_JOB) { + // const { jobId, threadId } = args as e.AcceptJobEvent; + // acceptJob(jobId, name, threadId); + // } else if (type === e.COMPLETE_JOB) { + // const { jobId, state } = args as e.CompleteJobEvent; + // completeJob(jobId, state); + // } + // }, + // }); + // return jobsList.get(thisJobId) as JobStats; + // } + // throw new Error('Job not found: ' + name); + // }; + + // register a job to enable it to be run + // The job will be compiled + // const registerJob = (name: string, source: string) => { + // if (registry[name]) { + // throw new Error('Job already registered: ' + name); + // } + // registry[name] = compile(source); + // }; + + // const getRegisteredJobs = () => Object.keys(registry); + + const getActiveJobs = (): JobStats[] => { + const jobs = activeJobs.map((id) => jobsList.get(id)); + return jobs.filter((j) => j) as JobStats[]; // no-op for typings + }; + + const getCompletedJobs = (): JobStats[] => { + return Array.from(jobsList.values()).filter((job) => job.status === 'done'); + }; + + const getErroredJobs = (): JobStats[] => { + return Array.from(jobsList.values()).filter((job) => job.status === 'err'); + }; + + return { + id, + on: events.on, + once: events.once, + execute: handleExecute, + // getStatus, // no tests on this yet, not sure if I want to commit to it + + getActiveJobs, + getCompletedJobs, + getErroredJobs, + + // TO REMOVE + // run, + // registerJob, + // getRegisteredJobs, + + // For testing + _registry: registry, + }; +}; + +export default createRTM; diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts new file mode 100644 index 000000000..00dc73d1b --- /dev/null +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -0,0 +1,10 @@ +// https://github.com/OpenFn/kit/issues/251 +const createAutoInstall = (options) => { + return async (plan) => { + // find all adaptors + // auto install what we need + // return when ready + }; +}; + +export default createAutoInstall; diff --git a/packages/runtime-manager/src/runners/compile.ts b/packages/runtime-manager/src/runners/compile.ts new file mode 100644 index 000000000..57340d6f7 --- /dev/null +++ b/packages/runtime-manager/src/runners/compile.ts @@ -0,0 +1,63 @@ +// This function will compile a workflow +// Later we'll add an in-memory cache to prevent the same job +// being compiled twice + +import type { Logger } from '@openfn/logger'; +import compile, { preloadAdaptorExports, Options } from '@openfn/compiler'; +import { getModulePath } from '@openfn/runtime'; + +const createCompile = (logger: Logger, repoDir: string) => { + const cache = {}; + return async (plan) => { + // Compile each job in the exeuction plan + // A bit like the CLI + for (const job of plan.jobs) { + if (job.expression) { + job.expression = await compileJob( + job.expression as string, + job.adaptor, // TODO need to expand this. Or do I? + repoDir, + logger + ); + } + } + return plan; + }; +}; + +export default createCompile; + +// TODO copied out of CLI +const stripVersionSpecifier = (specifier: string) => { + const idx = specifier.lastIndexOf('@'); + if (idx > 0) { + return specifier.substring(0, idx); + } + return specifier; +}; + +const compileJob = async ( + job: string, + adaptor: string, + repoDir: string, + logger: Logger +) => { + try { + // TODO I probably dont want to log this stuff + const pathToAdaptor = await getModulePath(adaptor, repoDir, logger); + const exports = await preloadAdaptorExports(pathToAdaptor!, false, logger); + const compilerOptions = { + logger, + ['add-imports']: { + adaptor: { + name: stripVersionSpecifier(adaptor), + exports, + exportAll: true, + }, + }, + }; + return compile(job, compilerOptions); + } catch (e: any) { + console.log(e); + } +}; diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts new file mode 100644 index 000000000..00616ae3e --- /dev/null +++ b/packages/runtime-manager/src/runners/execute.ts @@ -0,0 +1,31 @@ +// Execute a compiled workflow +import type { WorkerPool } from 'workerpool'; +import type { ExecutionPlan } from '@openfn/runtime'; +import * as e from '../events'; + +// A lot of callbacks needed here +// Is it better to just return the handler? +// But then this function really isn't doing so much +// (I guess that's true anyway) +const execute = (workers: WorkerPool, onAcceptJob?, onLog?, onError?) => { + return (plan: ExecutionPlan) => { + return new Promise((resolve) => { + console.log('executing...'); + workers.exec('run', [plan], { + on: ({ type, ...args }: e.JobEvent) => { + if (type === e.ACCEPT_JOB) { + console.log('accept'); + const { jobId, threadId } = args as e.AcceptJobEvent; + onAcceptJob?.(jobId, plan.id, threadId); + } else if (type === e.COMPLETE_JOB) { + console.log('complete'); + const { jobId, state } = args as e.CompleteJobEvent; + resolve(state); + } + }, + }); + }); + }; +}; + +export default execute; diff --git a/packages/runtime-manager/src/server/index.ts b/packages/runtime-manager/src/server/index.ts index 866104d2a..b27696466 100644 --- a/packages/runtime-manager/src/server/index.ts +++ b/packages/runtime-manager/src/server/index.ts @@ -1,7 +1,7 @@ import koa from 'koa'; import fs from 'node:fs/promises'; import path from 'node:path'; -import Manager from '../Manager'; +import Manager from '../rtm'; const loadJobs = async () => { for (const name of ['slow-random']) { diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index a1c935145..e5efc46ef 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -11,11 +11,11 @@ function publish(event: e.JobEvent) { // When the worker starts, it should report back its id // We need the runaround here because our worker pool obfuscates it -function init(jobId: number) { +function init(jobId: string) { publish({ type: e.ACCEPT_JOB, jobId, threadId }); } -async function helper(jobId: number, fn: () => Promise) { +async function helper(jobId: string, fn: () => Promise) { init(jobId); try { const result = await fn(); diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index a280c750d..d6753ecdb 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -2,13 +2,21 @@ // Security thoughts: the process inherits the node command arguments // (it has to for experimental modules to work) // Is this a concern? If secrets are passed in they could be visible -// The sandbox should hep +// The sandbox should help + +// What about imports in a worker thread? +// Is there an overjhead in reimporting stuff (presumably!) +// Should we actually be pooling workers by adaptor[+version] +// Does this increase the danger of sharing state between jobs? +// Suddenly it's a liability for the same environent in the same adaptor +// to be running the same jobs - break out of the sandbox and who knows what you can get import workerpool from 'workerpool'; import helper from './worker-helper'; import run from '@openfn/runtime'; +import type { ExecutionPlan } from '@openfn/runtime'; workerpool.worker({ - run: async (jobId: number, src: string, state?: any) => { - return helper(jobId, async () => run(src, state)); + run: async (plan: ExecutionPlan) => { + return helper(plan.id!, async () => run(plan)); }, }); diff --git a/packages/runtime-manager/test/manager.test.ts b/packages/runtime-manager/test/manager.test.ts index b981569f3..7771519f0 100644 --- a/packages/runtime-manager/test/manager.test.ts +++ b/packages/runtime-manager/test/manager.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import Manager from '../src/Manager'; +import Manager from '../src/rtm'; test('Should create a new manager', (t) => { const m = Manager(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943fa6902..6b29cf6dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -335,22 +335,31 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime + '@openfn/runtime-manager': + specifier: workspace:* + version: link:../runtime-manager koa: specifier: ^2.13.4 version: 2.13.4 koa-bodyparser: specifier: ^4.4.0 version: 4.4.0 - yargs: - specifier: ^17.5.1 - version: 17.7.2 devDependencies: '@types/koa': specifier: ^2.13.5 version: 2.13.5 + '@types/koa-bodyparser': + specifier: ^4.3.10 + version: 4.3.10 + '@types/koa-router': + specifier: ^7.4.4 + version: 7.4.4 '@types/node': specifier: ^18.15.3 version: 18.15.3 + '@types/yargs': + specifier: ^17.0.12 + version: 17.0.24 ava: specifier: 5.1.0 version: 5.1.0 @@ -363,15 +372,15 @@ importers: tslib: specifier: ^2.4.0 version: 2.4.0 - tsm: - specifier: ^2.2.2 - version: 2.2.2 tsup: specifier: ^6.2.3 version: 6.2.3(ts-node@10.9.1)(typescript@4.8.3) typescript: specifier: ^4.6.4 version: 4.8.3 + yargs: + specifier: ^17.6.2 + version: 17.7.2 packages/runtime: dependencies: @@ -1281,11 +1290,23 @@ packages: /@types/keygrip@1.0.2: resolution: {integrity: sha512-GJhpTepz2udxGexqos8wgaBx4I/zWIDPh/KOGEwAqtuGDkOUJu5eFvwmdBX4AmB8Odsr+9pHCQqiAqDL/yKMKw==} + /@types/koa-bodyparser@4.3.10: + resolution: {integrity: sha512-6ae05pjhmrmGhUR8GYD5qr5p9LTEMEGfGXCsK8VaSL+totwigm8+H/7MHW7K4854CMeuwRAubT8qcc/EagaeIA==} + dependencies: + '@types/koa': 2.13.5 + dev: true + /@types/koa-compose@3.2.5: resolution: {integrity: sha512-B8nG/OoE1ORZqCkBVsup/AKcvjdgoHnfi4pZMn5UwAPCbhk/96xyv284eBYW8JlQbQ7zDmnpFr68I/40mFoIBQ==} dependencies: '@types/koa': 2.13.5 + /@types/koa-router@7.4.4: + resolution: {integrity: sha512-3dHlZ6CkhgcWeF6wafEUvyyqjWYfKmev3vy1PtOmr0mBc3wpXPU5E8fBBd4YQo5bRpHPfmwC5yDaX7s4jhIN6A==} + dependencies: + '@types/koa': 2.13.5 + dev: true + /@types/koa@2.13.5: resolution: {integrity: sha512-HSUOdzKz3by4fnqagwthW/1w/yJspTgppyyalPVbgZf8jQWvdIXcVW5h2DGtw4zYntOaeRGx49r1hxoPWrD4aA==} dependencies: From 6f76923867ffba7a7e1a7d1891fbbbb409fdbc58 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 1 Jun 2023 13:10:57 +0100 Subject: [PATCH 019/232] runtime-manager: fix worker path --- packages/rtm-server/src/start.ts | 5 +-- packages/runtime-manager/package.json | 2 +- packages/runtime-manager/src/rtm.ts | 27 ++++++++-------- .../runtime-manager/src/runners/execute.ts | 31 +++++++++++-------- packages/runtime-manager/src/worker-helper.ts | 1 + packages/runtime-manager/src/worker.ts | 3 ++ 6 files changed, 39 insertions(+), 30 deletions(-) diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index 9dcc18518..9ab177f93 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -12,6 +12,8 @@ type Args = { lightning?: string; }; +const logger = createLogger('SRV'); + const args = yargs(hideBin(process.argv)) .command('server', 'Start a runtime manager server') .option('port', { @@ -27,10 +29,9 @@ const args = yargs(hideBin(process.argv)) .parse() as Args; const rtm = createRTM(); -console.log('RTM created'); +logger.debug('RTM created'); // TODO why is this blowing up?? -const logger = createLogger('SRV'); createRTMServer(rtm, { port: args.port, diff --git a/packages/runtime-manager/package.json b/packages/runtime-manager/package.json index ed61f69bf..6cf3b0ab7 100644 --- a/packages/runtime-manager/package.json +++ b/packages/runtime-manager/package.json @@ -8,7 +8,7 @@ "scripts": { "test": "pnpm ava", "test:types": "pnpm tsc --noEmit --project tsconfig.json", - "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", + "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", "build:watch": "pnpm build --watch", "serve": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/server/index.ts'" }, diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 1c3a6a868..9d463f479 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -1,11 +1,12 @@ import path from 'node:path'; import crypto from 'node:crypto'; +import { fileURLToPath } from 'url'; import { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; import { ExecutionPlan } from '@openfn/runtime'; -import * as e from './events'; -import createAutoinstall from './runners/autoinstall'; +// import * as e from './events'; +// import createAutoinstall from './runners/autoinstall'; import createCompile from './runners/compile'; import createExecute from './runners/execute'; @@ -16,12 +17,10 @@ type LiveJob = Array<(s: State) => State>; type JobRegistry = Record; -let jobid = 1000; - // Archive of every job we've run // Fien to just keep in memory for now type JobStats = { - id: number; + id: string; name: string; status: 'pending' | 'done' | 'err'; startTime: number; @@ -43,22 +42,23 @@ export type LazyResolvers = { const createRTM = function ( serverId?: string, - resolvers?: LazyResolvers, + _resolvers?: LazyResolvers, useMock = false ) { const id = serverId || crypto.randomUUID(); - const jobsList: Map = new Map(); - const activeJobs: number[] = []; + const jobsList: Map = new Map(); + const activeJobs: string[] = []; const registry: JobRegistry = {}; - const workers = workerpool.pool( - path.resolve(useMock ? './dist/mock-worker.js' : './dist/worker.js') - ); + + const dirname = path.dirname(fileURLToPath(import.meta.url)); + const p = path.resolve(dirname, useMock ? './mock-worker.js' : './worker.js'); + const workers = workerpool.pool(p); const events = new EventEmitter(); - const acceptJob = (jobId: number, name: string, threadId: number) => { + const acceptJob = (jobId: string, name: string, threadId: number) => { console.log('>> Accept job'); if (jobsList.has(jobId)) { throw new Error(`Job with id ${jobId} is already defined`); @@ -74,7 +74,7 @@ const createRTM = function ( activeJobs.push(jobId); }; - const completeJob = (jobId: number, state: any) => { + const completeJob = (jobId: string, state: any) => { console.log('>> complete job'); if (!jobsList.has(jobId)) { throw new Error(`Job with id ${jobId} is not defined`); @@ -98,7 +98,6 @@ const createRTM = function ( // autoinstall // compile it const compiledPlan = await compile(plan); - console.log(JSON.stringify(compiledPlan, null, 2)); const result = await execute(compiledPlan); console.log('RESULT', result); diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index 00616ae3e..e1240b900 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -11,19 +11,24 @@ const execute = (workers: WorkerPool, onAcceptJob?, onLog?, onError?) => { return (plan: ExecutionPlan) => { return new Promise((resolve) => { console.log('executing...'); - workers.exec('run', [plan], { - on: ({ type, ...args }: e.JobEvent) => { - if (type === e.ACCEPT_JOB) { - console.log('accept'); - const { jobId, threadId } = args as e.AcceptJobEvent; - onAcceptJob?.(jobId, plan.id, threadId); - } else if (type === e.COMPLETE_JOB) { - console.log('complete'); - const { jobId, state } = args as e.CompleteJobEvent; - resolve(state); - } - }, - }); + try { + workers.exec('run', [plan], { + on: ({ type, ...args }: e.JobEvent) => { + console.log('EVENT', type); + if (type === e.ACCEPT_JOB) { + console.log('accept'); + const { jobId, threadId } = args as e.AcceptJobEvent; + onAcceptJob?.(jobId, plan.id, threadId); + } else if (type === e.COMPLETE_JOB) { + console.log('complete'); + const { jobId, state } = args as e.CompleteJobEvent; + resolve(state); + } + }, + }); + } catch (e) { + console.log(e); + } }); }; }; diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index e5efc46ef..9fb119577 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -16,6 +16,7 @@ function init(jobId: string) { } async function helper(jobId: string, fn: () => Promise) { + console.log('worker helper'); init(jobId); try { const result = await fn(); diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index d6753ecdb..76d7875f7 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -15,8 +15,11 @@ import helper from './worker-helper'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; +console.log('LOADING WORKER'); + workerpool.worker({ run: async (plan: ExecutionPlan) => { + console.log('running worker'); return helper(plan.id!, async () => run(plan)); }, }); From eed07601175ba747ada04f5e6b575d7803a7fbbc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 1 Jun 2023 16:32:26 +0100 Subject: [PATCH 020/232] rtm: feed repoDir through to runtime and hook up execute properly --- packages/rtm-server/package.json | 8 +- .../rtm-server/src/middleware/workflow.ts | 10 ++- packages/rtm-server/src/server.ts | 6 ++ packages/rtm-server/yargs | 0 packages/runtime-manager/src/rtm.ts | 88 +++++++------------ .../runtime-manager/src/runners/compile.ts | 30 +++---- .../runtime-manager/src/runners/execute.ts | 31 ++++--- packages/runtime-manager/src/worker.ts | 23 +++-- pnpm-lock.yaml | 36 ++++++-- 9 files changed, 131 insertions(+), 101 deletions(-) create mode 100644 packages/rtm-server/yargs diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 32ffee358..d7185722d 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -10,8 +10,8 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", - "start": "ts-node-esm src/start.ts", - "_start": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/index.ts'" + "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm src/start.ts", + "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'" }, "author": "Open Function Group ", "license": "ISC", @@ -20,8 +20,10 @@ "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", "@openfn/runtime-manager": "workspace:*", + "@types/koa-logger": "^3.1.2", "koa": "^2.13.4", - "koa-bodyparser": "^4.4.0" + "koa-bodyparser": "^4.4.0", + "koa-logger": "^3.2.1" }, "devDependencies": { "@types/koa": "^2.13.5", diff --git a/packages/rtm-server/src/middleware/workflow.ts b/packages/rtm-server/src/middleware/workflow.ts index 78f35f23e..4c5781a71 100644 --- a/packages/rtm-server/src/middleware/workflow.ts +++ b/packages/rtm-server/src/middleware/workflow.ts @@ -1,9 +1,17 @@ +import crypto from 'node:crypto'; + export default (execute: any, logger: any) => (ctx: any) => { - logger.info('POST TO WORKFLOW'); try { const attempt = ctx.request.body; + if (!attempt.id) { + // This is really useful from a dev perspective + // If an attempt doesn't have an id, autogenerate one + logger.info('autogenerating id for incoming attempt'); + attempt.id = crypto.randomUUID(); + } // TODO should this return the result... ? // No other way to get hold of it + logger.info('Execute attempt ', attempt.id); execute(attempt); ctx.status = 200; } catch (e: any) { diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 813987fdd..c4ad975b4 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -9,6 +9,7 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; +import koaLogger from 'koa-logger'; import createAPI from './api'; import startWorkLoop from './work-loop'; @@ -76,6 +77,11 @@ function createServer(rtm: any, options: ServerOptions = {}) { const app = new Koa(); app.use(bodyParser()); + app.use( + koaLogger((str, _args) => { + logger.debug(str); + }) + ); const execute = (attempt: Attempt) => { const plan = convertAttempt(attempt); diff --git a/packages/rtm-server/yargs b/packages/rtm-server/yargs new file mode 100644 index 000000000..e69de29bb diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 9d463f479..64f42f8e9 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -9,6 +9,7 @@ import { ExecutionPlan } from '@openfn/runtime'; // import createAutoinstall from './runners/autoinstall'; import createCompile from './runners/compile'; import createExecute from './runners/execute'; +import createLogger, { Logger } from '@openfn/logger'; export type State = any; // TODO I want a nice state def with generics @@ -40,12 +41,19 @@ export type LazyResolvers = { expressions?: Resolver; }; -const createRTM = function ( - serverId?: string, - _resolvers?: LazyResolvers, - useMock = false -) { +type RTMOptions = { + resolvers: LazyResolvers; + logger: Logger; + useMock: false; + repoDir: string; +}; + +const createRTM = function (serverId?: string, options: RTMOptions = {}) { + const { resolvers, useMock } = options; + let { repoDir } = options; + const id = serverId || crypto.randomUUID(); + const logger = options.logger || createLogger('RTM', { level: 'debug' }); const jobsList: Map = new Map(); const activeJobs: string[] = []; @@ -58,10 +66,15 @@ const createRTM = function ( const events = new EventEmitter(); + if (!repoDir) { + repoDir = '/tmp/openfn/repo'; + logger.info('Defaulting repoDir to ', repoDir); + } + const acceptJob = (jobId: string, name: string, threadId: number) => { - console.log('>> Accept job'); + logger.info('accept job ', jobId); if (jobsList.has(jobId)) { - throw new Error(`Job with id ${jobId} is already defined`); + throw new Error(`Job with id ${jobId} is already in progress`); } jobsList.set(jobId, { id: jobId, @@ -75,7 +88,8 @@ const createRTM = function ( }; const completeJob = (jobId: string, state: any) => { - console.log('>> complete job'); + logger.success('complete job ', jobId); + logger.info(state); if (!jobsList.has(jobId)) { throw new Error(`Job with id ${jobId} is not defined`); } @@ -87,62 +101,32 @@ const createRTM = function ( activeJobs.splice(idx, 1); }; - const execute = createExecute(workers, acceptJob); - const compile = createCompile(console as any, '/tmp/openfn/repo'); + // Create "runner" functions for execute and compile + const execute = createExecute(workers, repoDir, logger, { + accept: acceptJob, + }); + const compile = createCompile(logger, repoDir); // How much of this happens inside the worker? // Shoud the main thread handle compilation? Has to if we want to cache // Unless we create a dedicated compiler worker // TODO error handling, timeout const handleExecute = async (plan: ExecutionPlan) => { - // autoinstall - // compile it - const compiledPlan = await compile(plan); + logger.debug('Executing plan ', plan.id); + // TODO autoinstall + + const compiledPlan = await compile(plan); + logger.debug('plan compiled ', plan.id); const result = await execute(compiledPlan); - console.log('RESULT', result); completeJob(plan.id!, result); + logger.debug('finished executing plan ', plan.id); // Return the result // Note that the mock doesn't behave like ths // And tbf I don't think we should keep the promise open - there's no point? return result; }; - - // // Run a job in a worker - // // Accepts the name of a registered job - // const run = async (name: string, state?: any): Promise => { - // const src = registry[name]; - // if (src) { - // const thisJobId = ++jobid; - - // await workers.exec('run', [jobid, src, state], { - // on: ({ type, ...args }: e.JobEvent) => { - // if (type === e.ACCEPT_JOB) { - // const { jobId, threadId } = args as e.AcceptJobEvent; - // acceptJob(jobId, name, threadId); - // } else if (type === e.COMPLETE_JOB) { - // const { jobId, state } = args as e.CompleteJobEvent; - // completeJob(jobId, state); - // } - // }, - // }); - // return jobsList.get(thisJobId) as JobStats; - // } - // throw new Error('Job not found: ' + name); - // }; - - // register a job to enable it to be run - // The job will be compiled - // const registerJob = (name: string, source: string) => { - // if (registry[name]) { - // throw new Error('Job already registered: ' + name); - // } - // registry[name] = compile(source); - // }; - - // const getRegisteredJobs = () => Object.keys(registry); - const getActiveJobs = (): JobStats[] => { const jobs = activeJobs.map((id) => jobsList.get(id)); return jobs.filter((j) => j) as JobStats[]; // no-op for typings @@ -167,12 +151,6 @@ const createRTM = function ( getCompletedJobs, getErroredJobs, - // TO REMOVE - // run, - // registerJob, - // getRegisteredJobs, - - // For testing _registry: registry, }; }; diff --git a/packages/runtime-manager/src/runners/compile.ts b/packages/runtime-manager/src/runners/compile.ts index 57340d6f7..11089c9c7 100644 --- a/packages/runtime-manager/src/runners/compile.ts +++ b/packages/runtime-manager/src/runners/compile.ts @@ -42,22 +42,18 @@ const compileJob = async ( repoDir: string, logger: Logger ) => { - try { - // TODO I probably dont want to log this stuff - const pathToAdaptor = await getModulePath(adaptor, repoDir, logger); - const exports = await preloadAdaptorExports(pathToAdaptor!, false, logger); - const compilerOptions = { - logger, - ['add-imports']: { - adaptor: { - name: stripVersionSpecifier(adaptor), - exports, - exportAll: true, - }, + // TODO I probably dont want to log this stuff + const pathToAdaptor = await getModulePath(adaptor, repoDir, logger); + const exports = await preloadAdaptorExports(pathToAdaptor!, false, logger); + const compilerOptions = { + logger, + ['add-imports']: { + adaptor: { + name: stripVersionSpecifier(adaptor), + exports, + exportAll: true, }, - }; - return compile(job, compilerOptions); - } catch (e: any) { - console.log(e); - } + }, + }; + return compile(job, compilerOptions); }; diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index e1240b900..b9c7e4cf5 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -1,35 +1,38 @@ // Execute a compiled workflow import type { WorkerPool } from 'workerpool'; import type { ExecutionPlan } from '@openfn/runtime'; +import { Logger } from '@openfn/logger'; import * as e from '../events'; // A lot of callbacks needed here // Is it better to just return the handler? // But then this function really isn't doing so much // (I guess that's true anyway) -const execute = (workers: WorkerPool, onAcceptJob?, onLog?, onError?) => { +const execute = ( + workers: WorkerPool, + repoDir: string, + logger: Logger, + events: any +) => { + const { accept, log, error } = events; return (plan: ExecutionPlan) => { - return new Promise((resolve) => { - console.log('executing...'); - try { - workers.exec('run', [plan], { + return new Promise((resolve) => + workers + .exec('run', [plan, repoDir], { on: ({ type, ...args }: e.JobEvent) => { - console.log('EVENT', type); if (type === e.ACCEPT_JOB) { - console.log('accept'); const { jobId, threadId } = args as e.AcceptJobEvent; - onAcceptJob?.(jobId, plan.id, threadId); + accept?.(jobId, plan.id, threadId); } else if (type === e.COMPLETE_JOB) { - console.log('complete'); const { jobId, state } = args as e.CompleteJobEvent; resolve(state); } }, - }); - } catch (e) { - console.log(e); - } - }); + }) + .catch((e) => { + logger.error(e); + }) + ); }; }; diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index 76d7875f7..1f9dbd2bb 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -1,3 +1,5 @@ +// Runs inside the worker + // Dedicated worker for running jobs // Security thoughts: the process inherits the node command arguments // (it has to for experimental modules to work) @@ -11,15 +13,26 @@ // Suddenly it's a liability for the same environent in the same adaptor // to be running the same jobs - break out of the sandbox and who knows what you can get import workerpool from 'workerpool'; -import helper from './worker-helper'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; +import createLogger from '@openfn/logger'; +import helper from './worker-helper'; -console.log('LOADING WORKER'); +// TODO how can we control the logger in here? +// Need some kind of intitialisation function to set names and levels +const logger = createLogger('R/T', { level: 'debug' }); +const jobLogger = createLogger('JOB', { level: 'debug' }); workerpool.worker({ - run: async (plan: ExecutionPlan) => { - console.log('running worker'); - return helper(plan.id!, async () => run(plan)); + run: (plan: ExecutionPlan, repoDir: string) => { + const options = { + logger, + jobLogger, + linker: { + repo: repoDir, + }, + }; + + return helper(plan.id!, () => run(plan, {}, options)); }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b29cf6dc..d34c260e1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -338,12 +338,18 @@ importers: '@openfn/runtime-manager': specifier: workspace:* version: link:../runtime-manager + '@types/koa-logger': + specifier: ^3.1.2 + version: 3.1.2 koa: specifier: ^2.13.4 version: 2.13.4 koa-bodyparser: specifier: ^4.4.0 version: 4.4.0 + koa-logger: + specifier: ^3.2.1 + version: 3.2.1 devDependencies: '@types/koa': specifier: ^2.13.5 @@ -1301,6 +1307,12 @@ packages: dependencies: '@types/koa': 2.13.5 + /@types/koa-logger@3.1.2: + resolution: {integrity: sha512-sioTA1xlKYiIgryANWPRHBkG3XGbWftw9slWADUPC+qvPIY/yRLSrhvX7zkJwMrntub5dPO0GuAoyGGf0yitfQ==} + dependencies: + '@types/koa': 2.13.5 + dev: false + /@types/koa-router@7.4.4: resolution: {integrity: sha512-3dHlZ6CkhgcWeF6wafEUvyyqjWYfKmev3vy1PtOmr0mBc3wpXPU5E8fBBd4YQo5bRpHPfmwC5yDaX7s4jhIN6A==} dependencies: @@ -1521,7 +1533,6 @@ packages: engines: {node: '>=4'} dependencies: color-convert: 1.9.3 - dev: true /ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} @@ -1947,7 +1958,6 @@ packages: ansi-styles: 3.2.1 escape-string-regexp: 1.0.5 supports-color: 5.5.0 - dev: true /chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} @@ -2102,7 +2112,6 @@ packages: resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} dependencies: color-name: 1.1.3 - dev: true /color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} @@ -2112,7 +2121,6 @@ packages: /color-name@1.1.3: resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - dev: true /color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} @@ -3532,7 +3540,6 @@ packages: /has-flag@3.0.0: resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} engines: {node: '>=4'} - dev: true /has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} @@ -3661,6 +3668,10 @@ packages: engines: {node: '>=10.17.0'} dev: true + /humanize-number@0.0.2: + resolution: {integrity: sha512-un3ZAcNQGI7RzaWGZzQDH47HETM4Wrj6z6E4TId8Yeq9w5ZKUVB1nrT2jwFheTUjEmqcgTjXDc959jum+ai1kQ==} + dev: false + /iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -4132,6 +4143,16 @@ packages: koa-compose: 4.1.0 dev: false + /koa-logger@3.2.1: + resolution: {integrity: sha512-MjlznhLLKy9+kG8nAXKJLM0/ClsQp/Or2vI3a5rbSQmgl8IJBQO0KI5FA70BvW+hqjtxjp49SpH2E7okS6NmHg==} + engines: {node: '>= 7.6.0'} + dependencies: + bytes: 3.1.2 + chalk: 2.4.2 + humanize-number: 0.0.2 + passthrough-counter: 1.0.0 + dev: false + /koa@2.13.4: resolution: {integrity: sha512-43zkIKubNbnrULWlHdN5h1g3SEKXOEzoAlRsHOTFpnlDu8JlAOZSMJBLULusuXRequboiwJcj5vtYXKB3k7+2g==} engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} @@ -4831,6 +4852,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /passthrough-counter@1.0.0: + resolution: {integrity: sha512-Wy8PXTLqPAN0oEgBrlnsXPMww3SYJ44tQ8aVrGAI4h4JZYCS0oYqsPqtPR8OhJpv6qFbpbB7XAn0liKV7EXubA==} + dev: false + /path-dirname@1.0.2: resolution: {integrity: sha512-ALzNPpyNq9AqXMBjeymIjFDAkAFH06mHJH/cSBHAgU0s4vfpBn6b2nf8tiRLvagKD8RbTpq2FKTBg7cl9l3c7Q==} dev: true @@ -5803,7 +5828,6 @@ packages: engines: {node: '>=4'} dependencies: has-flag: 3.0.0 - dev: true /supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} From 30552db86cf2e94148f80a0c6bf33de8f38d7cf2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 1 Jun 2023 19:30:56 +0100 Subject: [PATCH 021/232] rtm: refactor away from jobs --- packages/runtime-manager/src/events.ts | 23 ++-- packages/runtime-manager/src/rtm.ts | 120 +++++++++--------- .../runtime-manager/src/runners/execute.ts | 16 ++- packages/runtime-manager/src/worker-helper.ts | 19 +-- 4 files changed, 90 insertions(+), 88 deletions(-) diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index 34c8f2923..212dd60f7 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -1,27 +1,30 @@ -export const ACCEPT_JOB = 'accept-job'; +export const WORKFLOW_START = 'workflow-start'; -export const COMPLETE_JOB = 'complete-job'; +export const WORKFLOW_COMPLETE = 'workflow-complete'; -export const JOB_ERROR = 'job-error'; +export const WORKFLOW_ERROR = 'workflow-error'; type State = any; // TODO -export type AcceptJobEvent = { - type: typeof ACCEPT_JOB; +export type AcceptWorkflowEvent = { + type: typeof WORKFLOW_START; jobId: string; threadId: number; }; -export type CompleteJobEvent = { - type: typeof COMPLETE_JOB; +export type CompleteWorkflowEvent = { + type: typeof WORKFLOW_COMPLETE; jobId: string; state: State; }; -export type ErrJobEvent = { - type: typeof JOB_ERROR; +export type ErrWorkflowEvent = { + type: typeof WORKFLOW_ERROR; jobId: string; message: string; }; -export type JobEvent = AcceptJobEvent | CompleteJobEvent | ErrJobEvent; +export type WorkflowEvent = + | AcceptWorkflowEvent + | CompleteWorkflowEvent + | ErrWorkflowEvent; diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 64f42f8e9..66175dcb7 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -13,22 +13,18 @@ import createLogger, { Logger } from '@openfn/logger'; export type State = any; // TODO I want a nice state def with generics -// hmm, may need to support this for unit tests (which does kind of make sense) -type LiveJob = Array<(s: State) => State>; - -type JobRegistry = Record; - -// Archive of every job we've run -// Fien to just keep in memory for now -type JobStats = { +// Archive of every workflow we've run +// Fine to just keep in memory for now +type WorkflowStats = { id: string; - name: string; + name?: string; // TODO what is name? this is irrelevant? status: 'pending' | 'done' | 'err'; - startTime: number; - threadId: number; - duration: number; + startTime?: number; + threadId?: number; + duration?: number; error?: string; result?: any; // State + plan: ExecutionPlan; }; type Resolver = (id: string) => Promise; @@ -55,10 +51,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const id = serverId || crypto.randomUUID(); const logger = options.logger || createLogger('RTM', { level: 'debug' }); - const jobsList: Map = new Map(); - const activeJobs: string[] = []; - - const registry: JobRegistry = {}; + const allWorkflows: Map = new Map(); + const activeWorkflows: string[] = []; const dirname = path.dirname(fileURLToPath(import.meta.url)); const p = path.resolve(dirname, useMock ? './mock-worker.js' : './worker.js'); @@ -71,39 +65,40 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { logger.info('Defaulting repoDir to ', repoDir); } - const acceptJob = (jobId: string, name: string, threadId: number) => { - logger.info('accept job ', jobId); - if (jobsList.has(jobId)) { - throw new Error(`Job with id ${jobId} is already in progress`); + const onWorkflowStarted = (workflowId: string, threadId: number) => { + logger.info('starting workflow ', workflowId); + + const workflow = allWorkflows.get(workflowId)!; + if (workflow.startTime) { + // TODO this shouldn't throw.. but what do we do? + // We shouldn't run a workflow that's been run + // Every workflow should have a unique id + // maybe the RTM doesn't care about this + throw new Error(`Workflow with id ${workflowId} is already started`); } - jobsList.set(jobId, { - id: jobId, - name, - status: 'pending', - threadId, - startTime: new Date().getTime(), - duration: -1, - }); - activeJobs.push(jobId); + workflow.startTime = new Date().getTime(); + workflow.duration = -1; + workflow.threadId = threadId; + activeWorkflows.push(workflowId); }; - const completeJob = (jobId: string, state: any) => { - logger.success('complete job ', jobId); + const completeWorkflow = (workflowId: string, state: any) => { + logger.success('complete workflow ', workflowId); logger.info(state); - if (!jobsList.has(jobId)) { - throw new Error(`Job with id ${jobId} is not defined`); + if (!allWorkflows.has(workflowId)) { + throw new Error(`Workflow with id ${workflowId} is not defined`); } - const job = jobsList.get(jobId)!; - job.status = 'done'; - job.result = state; - job.duration = new Date().getTime() - job.startTime; - const idx = activeJobs.findIndex((id) => id === jobId); - activeJobs.splice(idx, 1); + const workflow = allWorkflows.get(workflowId)!; + workflow.status = 'done'; + workflow.result = state; + workflow.duration = new Date().getTime() - workflow.startTime; + const idx = activeWorkflows.findIndex((id) => id === workflowId); + activeWorkflows.splice(idx, 1); }; // Create "runner" functions for execute and compile const execute = createExecute(workers, repoDir, logger, { - accept: acceptJob, + start: onWorkflowStarted, }); const compile = createCompile(logger, repoDir); @@ -112,33 +107,42 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { // Unless we create a dedicated compiler worker // TODO error handling, timeout const handleExecute = async (plan: ExecutionPlan) => { - logger.debug('Executing plan ', plan.id); + logger.debug('Executing workflow ', plan.id); + + allWorkflows.set(plan.id, { + id: plan.id, + name: plan.name, + status: 'pending', + plan, + }); // TODO autoinstall const compiledPlan = await compile(plan); - logger.debug('plan compiled ', plan.id); + logger.debug('workflow compiled ', plan.id); const result = await execute(compiledPlan); - completeJob(plan.id!, result); + logger.success(result); + completeWorkflow(plan.id!, result); - logger.debug('finished executing plan ', plan.id); + logger.debug('finished executing workflow ', plan.id); // Return the result // Note that the mock doesn't behave like ths // And tbf I don't think we should keep the promise open - there's no point? return result; }; - const getActiveJobs = (): JobStats[] => { - const jobs = activeJobs.map((id) => jobsList.get(id)); - return jobs.filter((j) => j) as JobStats[]; // no-op for typings - }; - const getCompletedJobs = (): JobStats[] => { - return Array.from(jobsList.values()).filter((job) => job.status === 'done'); - }; + // const getActiveJobs = (): WorkflowStats[] => { + // const jobs = allWorkflows.map((id) => workflowList.get(id)); + // return jobs.filter((j) => j) as WorkflowStats[]; // no-op for typings + // }; - const getErroredJobs = (): JobStats[] => { - return Array.from(jobsList.values()).filter((job) => job.status === 'err'); - }; + // const getCompletedJobs = (): WorkflowStats[] => { + // return Array.from(allWorkflows.values()).filter((workflow) => workflow.status === 'done'); + // }; + + // const getErroredJobs = (): WorkflowStats[] => { + // return Array.from(workflowsList.values()).filter((workflow) => workflow.status === 'err'); + // }; return { id, @@ -147,11 +151,9 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { execute: handleExecute, // getStatus, // no tests on this yet, not sure if I want to commit to it - getActiveJobs, - getCompletedJobs, - getErroredJobs, - - _registry: registry, + // getActiveJobs, + // getCompletedJobs, + // getErroredJobs, }; }; diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index b9c7e4cf5..b07a112f8 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -14,17 +14,19 @@ const execute = ( logger: Logger, events: any ) => { - const { accept, log, error } = events; + const { start, log, error } = events; return (plan: ExecutionPlan) => { return new Promise((resolve) => workers .exec('run', [plan, repoDir], { - on: ({ type, ...args }: e.JobEvent) => { - if (type === e.ACCEPT_JOB) { - const { jobId, threadId } = args as e.AcceptJobEvent; - accept?.(jobId, plan.id, threadId); - } else if (type === e.COMPLETE_JOB) { - const { jobId, state } = args as e.CompleteJobEvent; + on: ({ type, ...args }: e.WorkflowEvent) => { + console.log(' >>>>>> ', type); + if (type === e.WORKFLOW_START) { + const { jobId, threadId } = args as e.AcceptWorkflowEvent; + start?.(jobId, threadId); + } else if (type === e.WORKFLOW_COMPLETE) { + console.log(' *** '); + const { jobId, state } = args as e.CompleteWorkflowEvent; resolve(state); } }, diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index 9fb119577..af1a91b76 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -5,27 +5,22 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; import * as e from './events'; -function publish(event: e.JobEvent) { +function publish(event: e.WorkflowEvent) { workerpool.workerEmit(event); } -// When the worker starts, it should report back its id -// We need the runaround here because our worker pool obfuscates it -function init(jobId: string) { - publish({ type: e.ACCEPT_JOB, jobId, threadId }); -} - async function helper(jobId: string, fn: () => Promise) { - console.log('worker helper'); - init(jobId); + publish({ type: e.WORKFLOW_START, jobId, threadId }); try { + // Note that the worker thread may fire logs after completion + // I think this is fine, it's jsut a log stream thing + // But the output is very confusing! const result = await fn(); - publish({ type: e.COMPLETE_JOB, jobId, state: result }); - return result; + publish({ type: e.WORKFLOW_COMPLETE, jobId, state: result }); } catch (err) { console.error(err); // @ts-ignore TODO sort out error typing - publish({ type: e.JOB_ERROR, jobId, message: err.message }); + publish({ type: e.WORKFLOW_ERROR, jobId, message: err.message }); } } From b67a6ac24148067b4a00b96bbf91cd1695daa233 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 2 Jun 2023 16:32:47 +0100 Subject: [PATCH 022/232] rtm: rewrite mock worker --- packages/runtime-manager/src/mock-worker.ts | 53 +++++--- packages/runtime-manager/src/rtm.ts | 5 +- .../runtime-manager/src/runners/execute.ts | 2 - packages/runtime-manager/src/worker-helper.ts | 3 + .../runtime-manager/test/mock-worker.test.ts | 114 +++++++++++++++--- 5 files changed, 138 insertions(+), 39 deletions(-) diff --git a/packages/runtime-manager/src/mock-worker.ts b/packages/runtime-manager/src/mock-worker.ts index 45b872616..9b090fef4 100644 --- a/packages/runtime-manager/src/mock-worker.ts +++ b/packages/runtime-manager/src/mock-worker.ts @@ -8,27 +8,50 @@ * and reading instructions out of state object. */ import workerpool from 'workerpool'; - -// Yeah not sure this import is right import helper from './worker-helper'; -const defaultArgs = { - returnValue: 42, - throw: undefined, // an error to throw - timeout: 0, // a timeout to wait before throwing or returning +type MockJob = { + id?: string; + adaptor?: string; + configuration?: any; + + expression?: string; // will evaluate as JSON + data?: any; // data will be returned if there's no expression + + // MS to delay the return by (underscored because it's a mock property) + _delay?: number; }; -async function mock(args = defaultArgs) { - const actualArgs = { - ...defaultArgs, - ...args, - }; +type MockExecutionPlan = { + id: string; + jobs: MockJob[]; +}; - return actualArgs.returnValue; +// This is a fake runtime handler which will return a fixed value, throw, and +// optionally delay +function mock(plan: MockExecutionPlan) { + const [job] = plan.jobs; + return new Promise((resolve) => { + setTimeout(() => { + let state: any = { data: job.data || {} }; + if (job.expression) { + try { + state = JSON.parse(job.expression); + } catch (e) { + state = { + data: job.data || {}, + error: { + [job.id || 'job']: e.message, + }, + }; + } + } + resolve(state); + }, job._delay || 1); + }); } workerpool.worker({ - run: async (jobId, _src, state) => { - return helper(jobId, async () => mock(state)); - }, + run: async (plan: MockExecutionPlan, _repoDir?: string) => + helper(plan.id, () => mock(plan)), }); diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 66175dcb7..9f676561a 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -109,9 +109,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const handleExecute = async (plan: ExecutionPlan) => { logger.debug('Executing workflow ', plan.id); - allWorkflows.set(plan.id, { - id: plan.id, - name: plan.name, + allWorkflows.set(plan.id!, { + id: plan.id!, status: 'pending', plan, }); diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index b07a112f8..2cb9452f4 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -20,12 +20,10 @@ const execute = ( workers .exec('run', [plan, repoDir], { on: ({ type, ...args }: e.WorkflowEvent) => { - console.log(' >>>>>> ', type); if (type === e.WORKFLOW_START) { const { jobId, threadId } = args as e.AcceptWorkflowEvent; start?.(jobId, threadId); } else if (type === e.WORKFLOW_COMPLETE) { - console.log(' *** '); const { jobId, state } = args as e.CompleteWorkflowEvent; resolve(state); } diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index af1a91b76..7516ad8cf 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -17,6 +17,9 @@ async function helper(jobId: string, fn: () => Promise) { // But the output is very confusing! const result = await fn(); publish({ type: e.WORKFLOW_COMPLETE, jobId, state: result }); + + // For tests + return result; } catch (err) { console.error(err); // @ts-ignore TODO sort out error typing diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index fead1f4e8..df56ade2b 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -1,36 +1,112 @@ /** * Simple suite of unit tests against the mock worker API - * Passes state in and expects the mock worker to behave as instructed - * Ie return a value, timeout, throw + * Passes a job in and expects to get a simulated result back * - * This file exercises the actual mock function, not the helper API - * TODO so I suppose it should text the mock itself, not the worker-wrapped one + * The mock is a real web worker which internally uses a mock runtime engine + * Inside the worker, everything apart from execute is the same as the real environment */ import path from 'node:path'; import test from 'ava'; import workerpool from 'workerpool'; +import * as e from '../src/events'; + const workers = workerpool.pool(path.resolve('dist/mock-worker.js')); -const jobid = 1; -const src = 'mock'; +const createPlan = (job?: {}) => ({ + id: 'wf-1', + jobs: [ + job || { + id: 'j1', + adaptor: 'common', // not used + credential: {}, // not used + data: {}, // Used if no expression + expression: JSON.stringify({ data: { answer: 42 } }), // Will be parsed + _delay: 1, // only used in the mock + }, + ], +}); + +test('execute a mock plan inside a worker thread', async (t) => { + const plan = createPlan(); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { answer: 42 } }); +}); -test('return a default value', async (t) => { - const state = {}; - const result = await workers.exec('run', [jobid, src, state]); - t.assert(result == 42); +test('execute a mock plan with data', async (t) => { + const plan = createPlan({ + id: 'j2', + data: { answer: 44 }, + }); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { answer: 44 } }); }); -test('return a simple value', async (t) => { - const state = { - returnValue: 10, - }; - const result = await workers.exec('run', [jobid, src, state]); - t.assert(result == 10); +test('execute a mock plan with an expression', async (t) => { + const plan = createPlan({ + id: 'j2', + expression: JSON.stringify({ data: { answer: 46 } }), + }); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { answer: 46 } }); }); -// should throw +test('expression state overrides data', async (t) => { + const plan = createPlan({ + id: 'j2', + data: { answer: 44 }, + expression: JSON.stringify({ data: { agent: '007' } }), + }); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { agent: '007' } }); +}); -// should return after a timeout +test('write an exception to state', async (t) => { + const plan = createPlan({ + id: 'j2', + expression: 'ƸӜƷ', // it's a butterfly, obviously (and mmore importantly, invalid JSON) + }); + const result = await workers.exec('run', [plan]); + t.truthy(result.data); + t.truthy(result.error); +}); -// should throw after a timeout +test('execute a mock plan with delay', async (t) => { + const start = new Date().getTime(); + const plan = createPlan({ + id: 'j1', + _delay: 50, + }); + await workers.exec('run', [plan]); + const elapsed = new Date().getTime() - start; + t.assert(elapsed > 50); +}); + +test('Publish workflow-start event', async (t) => { + const plan = createPlan(); + let didFire = false; + await workers.exec('run', [plan], { + on: ({ type, ...args }) => { + if (type === e.WORKFLOW_START) { + didFire = true; + } + }, + }); + t.true(didFire); +}); + +test('Publish workflow-complete event with state', async (t) => { + const plan = createPlan(); + let didFire = false; + let state; + await workers.exec('run', [plan], { + on: ({ type, ...args }) => { + if (type === e.WORKFLOW_COMPLETE) { + didFire = true; + state = args.state; + } + }, + }); + t.true(didFire); + t.deepEqual(state, { data: { answer: 42 } }); +}); From 17265ed3f54c2f4d7e50b79dad653cc7f62798a9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 2 Jun 2023 18:06:24 +0100 Subject: [PATCH 023/232] rtm: More tests and eventing --- packages/runtime-manager/src/rtm.ts | 50 +++++++--- packages/runtime-manager/test/manager.test.ts | 51 ---------- .../runtime-manager/test/mock-worker.test.ts | 16 +--- packages/runtime-manager/test/rtm.test.ts | 93 +++++++++++++++++++ packages/runtime-manager/test/util.ts | 15 +++ 5 files changed, 147 insertions(+), 78 deletions(-) delete mode 100644 packages/runtime-manager/test/manager.test.ts create mode 100644 packages/runtime-manager/test/rtm.test.ts create mode 100644 packages/runtime-manager/test/util.ts diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 9f676561a..49c31f5af 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -5,7 +5,7 @@ import { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; import { ExecutionPlan } from '@openfn/runtime'; -// import * as e from './events'; +import * as e from './events'; // import createAutoinstall from './runners/autoinstall'; import createCompile from './runners/compile'; import createExecute from './runners/execute'; @@ -38,15 +38,16 @@ export type LazyResolvers = { }; type RTMOptions = { - resolvers: LazyResolvers; - logger: Logger; - useMock: false; - repoDir: string; + resolvers?: LazyResolvers; + logger?: Logger; + workerPath?: string; + repoDir?: string; + noCompile: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? }; const createRTM = function (serverId?: string, options: RTMOptions = {}) { - const { resolvers, useMock } = options; - let { repoDir } = options; + const { resolvers, noCompile } = options; + let { repoDir, workerPath } = options; const id = serverId || crypto.randomUUID(); const logger = options.logger || createLogger('RTM', { level: 'debug' }); @@ -54,9 +55,17 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const allWorkflows: Map = new Map(); const activeWorkflows: string[] = []; - const dirname = path.dirname(fileURLToPath(import.meta.url)); - const p = path.resolve(dirname, useMock ? './mock-worker.js' : './worker.js'); - const workers = workerpool.pool(p); + let resolvedWorkerPath; + if (workerPath) { + // If a path to the worker has been passed in, just use it verbatim + // We use this to pass a mock worker for testing purposes + resolvedWorkerPath = workerPath; + } else { + // By default, we load ./worker.js but can't rely on the working dir to find it + const dirname = path.dirname(fileURLToPath(import.meta.url)); + resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + } + const workers = workerpool.pool(resolvedWorkerPath); const events = new EventEmitter(); @@ -80,6 +89,12 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { workflow.duration = -1; workflow.threadId = threadId; activeWorkflows.push(workflowId); + + // forward the event on to any external listeners + events.emit(e.WORKFLOW_START, { + workflowId, + // Should we publish anything else here? + }); }; const completeWorkflow = (workflowId: string, state: any) => { @@ -94,6 +109,13 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { workflow.duration = new Date().getTime() - workflow.startTime; const idx = activeWorkflows.findIndex((id) => id === workflowId); activeWorkflows.splice(idx, 1); + + // forward the event on to any external listeners + events.emit(e.WORKFLOW_COMPLETE, { + workflowId, + duration: workflow.duration, + state, + }); }; // Create "runner" functions for execute and compile @@ -117,7 +139,9 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { // TODO autoinstall - const compiledPlan = await compile(plan); + // Don't compile if we're running a mock (not a fan of this) + const compiledPlan = noCompile ? plan : await compile(plan); + logger.debug('workflow compiled ', plan.id); const result = await execute(compiledPlan); logger.success(result); @@ -145,8 +169,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { return { id, - on: events.on, - once: events.once, + on: (type: string, fn: (...args: any[]) => void) => events.on(type, fn), + once: (type: string, fn: (...args: any[]) => void) => events.once(type, fn), execute: handleExecute, // getStatus, // no tests on this yet, not sure if I want to commit to it diff --git a/packages/runtime-manager/test/manager.test.ts b/packages/runtime-manager/test/manager.test.ts deleted file mode 100644 index 7771519f0..000000000 --- a/packages/runtime-manager/test/manager.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import test from 'ava'; -import Manager from '../src/rtm'; - -test('Should create a new manager', (t) => { - const m = Manager(); - t.assert(m); - t.assert(m.run); -}); - -test('Should register a job', (t) => { - const m = Manager(); - m.registerJob('my_job', 'x'); - t.assert(m.getRegisteredJobs().includes('my_job')); -}); - -test('Should compile a registered job', (t) => { - const m = Manager(); - m.registerJob('my_job', 'fn()'); - - const compiled = m._registry['my_job']; - t.assert(compiled === 'export default [fn()];'); -}); - -test('Should throw if registering a job that already exists', (t) => { - const m = Manager(); - m.registerJob('my_job', 'x'); - t.throws(() => m.registerJob('my_job', 'x')); -}); - -test('Should return a registered job list', (t) => { - const m = Manager(); - m.registerJob('my_job', 'x'); - m.registerJob('my_other_job', 'x'); - - t.deepEqual(m.getRegisteredJobs(), ['my_job', 'my_other_job']); -}); - -test('Should run a mock job with a simple return value', async (t) => { - // This uses the mock worker, not the actual runtime - // It will still exercise all the lifecycle logic found in the worker-helper, - // Just not the runtime logic - const m = Manager(true); - m.registerJob('test', 'mock'); - const { result } = await m.run('test', { returnValue: 111 }); - t.assert(result === 111); -}); - -// should publish an event when a job starts -// should publish an event when a job stops -// should return a job list -// should return a list of active jobs diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index df56ade2b..ee32a1045 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -9,24 +9,12 @@ import path from 'node:path'; import test from 'ava'; import workerpool from 'workerpool'; +import { createPlan } from './util'; + import * as e from '../src/events'; const workers = workerpool.pool(path.resolve('dist/mock-worker.js')); -const createPlan = (job?: {}) => ({ - id: 'wf-1', - jobs: [ - job || { - id: 'j1', - adaptor: 'common', // not used - credential: {}, // not used - data: {}, // Used if no expression - expression: JSON.stringify({ data: { answer: 42 } }), // Will be parsed - _delay: 1, // only used in the mock - }, - ], -}); - test('execute a mock plan inside a worker thread', async (t) => { const plan = createPlan(); const result = await workers.exec('run', [plan]); diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts new file mode 100644 index 000000000..a85a2241a --- /dev/null +++ b/packages/runtime-manager/test/rtm.test.ts @@ -0,0 +1,93 @@ +import test from 'ava'; +import path from 'node:path'; +import { createMockLogger } from '@openfn/logger'; +import { createPlan } from './util'; + +import Manager from '../src/rtm'; +import * as e from '../src/events'; + +const logger = createMockLogger('', { level: 'debug' }); + +const options = { + // This uses the mock worker, not the actual runtime + // It will still exercise all the lifecycle logic found in the worker-helper, + // Just not the runtime logic + workerPath: path.resolve('dist/mock-worker.js'), + logger, + repoDir: '', // doesn't matter for the mock + noCompile: true, // messy - needed to allow an expression to be passed as json +}; + +test.afterEach(() => { + logger._reset(); +}); + +test('Should create a new manager', (t) => { + const rtm = Manager('x', options); + t.truthy(rtm); + t.truthy(rtm.execute); + t.truthy(rtm.on); + t.truthy(rtm.once); + t.is(rtm.id, 'x'); +}); + +test('Should run a mock job with a simple return value', async (t) => { + const state = { data: { x: 1 } }; + const rtm = Manager('x', options); + const plan = createPlan({ + expression: JSON.stringify(state), + }); + const result = await rtm.execute(plan); + t.deepEqual(result, state); +}); + +test('Should not explode if no adaptor is passed', async (t) => { + const state = { data: { x: 1 } }; + const rtm = Manager('x', options); + const plan = createPlan({ + expression: JSON.stringify(state), + }); + + // @ts-ignore + delete plan.jobs[0].adaptor; + const result = await rtm.execute(plan); + t.deepEqual(result, state); +}); + +test('events: workflow-start', async (t) => { + const rtm = Manager('x', options); + + let id; + let didCall; + rtm.on(e.WORKFLOW_START, ({ workflowId }) => { + didCall = true; + id = workflowId; + }); + + const plan = createPlan(); + await rtm.execute(plan); + + t.true(didCall); + t.is(id, plan.id); +}); + +test('events: workflow-complete', async (t) => { + const rtm = Manager('x', options); + + let didCall; + let evt; + rtm.on(e.WORKFLOW_COMPLETE, (e) => { + didCall = true; + evt = e; + }); + + const plan = createPlan(); + await rtm.execute(plan); + + t.true(didCall); + t.is(evt.workflowId, plan.id); + t.truthy(evt.duration); + t.deepEqual(evt.state, { data: { answer: 42 } }); +}); + +// TODO events: logging. How will I test this with the mock? diff --git a/packages/runtime-manager/test/util.ts b/packages/runtime-manager/test/util.ts new file mode 100644 index 000000000..1334245e5 --- /dev/null +++ b/packages/runtime-manager/test/util.ts @@ -0,0 +1,15 @@ +export const createPlan = (job = {}) => ({ + id: 'wf-1', + jobs: [ + { + id: 'j1', + adaptor: 'common', // not used + credential: {}, // not used + data: {}, // Used if no expression + expression: JSON.stringify({ data: { answer: 42 } }), // Will be parsed + _delay: 1, // only used in the mock + + ...job, + }, + ], +}); From a2a9f342b938e6e020bbdafc765ebcb1f7d02aea Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 12:18:08 +0100 Subject: [PATCH 024/232] rtm: use eval instead of json expressions --- packages/runtime-manager/src/mock-worker.ts | 10 +++++-- .../runtime-manager/test/mock-worker.test.ts | 30 +++++++++++++++++-- packages/runtime-manager/test/rtm.test.ts | 4 +-- packages/runtime-manager/test/util.ts | 2 +- 4 files changed, 38 insertions(+), 8 deletions(-) diff --git a/packages/runtime-manager/src/mock-worker.ts b/packages/runtime-manager/src/mock-worker.ts index 9b090fef4..9708ae7f0 100644 --- a/packages/runtime-manager/src/mock-worker.ts +++ b/packages/runtime-manager/src/mock-worker.ts @@ -32,11 +32,17 @@ type MockExecutionPlan = { function mock(plan: MockExecutionPlan) { const [job] = plan.jobs; return new Promise((resolve) => { - setTimeout(() => { + setTimeout(async () => { + // TODO this isn't data, but state - it's the whole state object (minus config) let state: any = { data: job.data || {} }; if (job.expression) { try { - state = JSON.parse(job.expression); + // Security considerations of eval here? + // If someone setup an rtm with the mock worker enabled, + // then all job code would be actually evalled + // To be fair, actual jobs wouldn't run, so it's not like anyone can run a malicious proxy server + const fn = eval(job.expression); + state = await fn(state); } catch (e) { state = { data: job.data || {}, diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index ee32a1045..7b3a1ae75 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -24,7 +24,7 @@ test('execute a mock plan inside a worker thread', async (t) => { test('execute a mock plan with data', async (t) => { const plan = createPlan({ id: 'j2', - data: { answer: 44 }, + data: { input: 44 }, }); const result = await workers.exec('run', [plan]); t.deepEqual(result, { data: { answer: 44 } }); @@ -33,7 +33,31 @@ test('execute a mock plan with data', async (t) => { test('execute a mock plan with an expression', async (t) => { const plan = createPlan({ id: 'j2', - expression: JSON.stringify({ data: { answer: 46 } }), + expression: '() => ({ data: { answer: 46 } })', + }); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { answer: 46 } }); +}); + +test('execute a mock plan with an expression which uses state', async (t) => { + const plan = createPlan({ + id: 'j2', + data: { input: 2 }, + expression: '(s) => ({ data: { answer: s.data.input * 2 } })', + }); + const result = await workers.exec('run', [plan]); + t.deepEqual(result, { data: { answer: 4 } }); +}); + +test('execute a mock plan with a promise expression', async (t) => { + const plan = createPlan({ + id: 'j2', + expression: `(s) => + new Promise((resolve) => { + setTimeout(() => { + resolve({ data: { answer: 46 } }) + }, 1); + })`, }); const result = await workers.exec('run', [plan]); t.deepEqual(result, { data: { answer: 46 } }); @@ -43,7 +67,7 @@ test('expression state overrides data', async (t) => { const plan = createPlan({ id: 'j2', data: { answer: 44 }, - expression: JSON.stringify({ data: { agent: '007' } }), + expression: '() => ({ data: { agent: "007" } })', }); const result = await workers.exec('run', [plan]); t.deepEqual(result, { data: { agent: '007' } }); diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts index a85a2241a..e4cb18dd2 100644 --- a/packages/runtime-manager/test/rtm.test.ts +++ b/packages/runtime-manager/test/rtm.test.ts @@ -35,7 +35,7 @@ test('Should run a mock job with a simple return value', async (t) => { const state = { data: { x: 1 } }; const rtm = Manager('x', options); const plan = createPlan({ - expression: JSON.stringify(state), + expression: `() => (${JSON.stringify(state)})`, }); const result = await rtm.execute(plan); t.deepEqual(result, state); @@ -45,7 +45,7 @@ test('Should not explode if no adaptor is passed', async (t) => { const state = { data: { x: 1 } }; const rtm = Manager('x', options); const plan = createPlan({ - expression: JSON.stringify(state), + expression: `() => (${JSON.stringify(state)})`, }); // @ts-ignore diff --git a/packages/runtime-manager/test/util.ts b/packages/runtime-manager/test/util.ts index 1334245e5..0777af17a 100644 --- a/packages/runtime-manager/test/util.ts +++ b/packages/runtime-manager/test/util.ts @@ -6,7 +6,7 @@ export const createPlan = (job = {}) => ({ adaptor: 'common', // not used credential: {}, // not used data: {}, // Used if no expression - expression: JSON.stringify({ data: { answer: 42 } }), // Will be parsed + expression: '(s) => ({ data: { answer: s.data?.input || 42 } })', _delay: 1, // only used in the mock ...job, From 3dcc1c9d6c44876a288bea10ab7a534df42934f4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 14:52:40 +0100 Subject: [PATCH 025/232] rtm: feed logs through the worker --- packages/runtime-manager/src/events.ts | 11 +++++- packages/runtime-manager/src/mock-worker.ts | 10 ++++-- packages/runtime-manager/src/worker-helper.ts | 35 +++++++++++++++++-- packages/runtime-manager/src/worker.ts | 10 ++---- .../runtime-manager/test/mock-worker.test.ts | 25 +++++++++++++ 5 files changed, 79 insertions(+), 12 deletions(-) diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index 212dd60f7..fdfb4510c 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -4,6 +4,8 @@ export const WORKFLOW_COMPLETE = 'workflow-complete'; export const WORKFLOW_ERROR = 'workflow-error'; +export const JOB_LOG = 'job-log'; + type State = any; // TODO export type AcceptWorkflowEvent = { @@ -24,7 +26,14 @@ export type ErrWorkflowEvent = { message: string; }; +export type JobLogEvent = { + type: typeof JOB_LOG; + jobId: string; + message: any; // TOD JSONLOG +}; + export type WorkflowEvent = | AcceptWorkflowEvent | CompleteWorkflowEvent - | ErrWorkflowEvent; + | ErrWorkflowEvent + | JobLogEvent; diff --git a/packages/runtime-manager/src/mock-worker.ts b/packages/runtime-manager/src/mock-worker.ts index 9708ae7f0..aa3696e3b 100644 --- a/packages/runtime-manager/src/mock-worker.ts +++ b/packages/runtime-manager/src/mock-worker.ts @@ -8,7 +8,9 @@ * and reading instructions out of state object. */ import workerpool from 'workerpool'; -import helper from './worker-helper'; +import helper, { createLoggers } from './worker-helper'; + +const { jobLogger } = createLoggers(); type MockJob = { id?: string; @@ -41,7 +43,11 @@ function mock(plan: MockExecutionPlan) { // If someone setup an rtm with the mock worker enabled, // then all job code would be actually evalled // To be fair, actual jobs wouldn't run, so it's not like anyone can run a malicious proxy server - const fn = eval(job.expression); + + // Override the console in the expression scope + const fn = new Function('console', 'return ' + job.expression)( + jobLogger + ); state = await fn(state); } catch (e) { state = { diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index 7516ad8cf..7deb01d0f 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -3,19 +3,50 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; +import createLogger from '@openfn/logger'; + import * as e from './events'; function publish(event: e.WorkflowEvent) { workerpool.workerEmit(event); } -async function helper(jobId: string, fn: () => Promise) { +export const createLoggers = () => { + const log = (jsonLog: any) => { + publish({ type: e.JOB_LOG, message: JSON.parse(jsonLog) }); + }; + + const emitter = { + info: log, + debug: log, + log, + warn: log, + error: log, + success: log, + always: log, + }; + + const logger = createLogger('R/T', { + logger: emitter, + level: 'debug', + json: true, + }); + const jobLogger = createLogger('JOB', { + logger: emitter, + level: 'debug', + json: true, + }); + + return { logger, jobLogger }; +}; + +async function helper(jobId: string, execute: () => Promise) { publish({ type: e.WORKFLOW_START, jobId, threadId }); try { // Note that the worker thread may fire logs after completion // I think this is fine, it's jsut a log stream thing // But the output is very confusing! - const result = await fn(); + const result = await execute(); publish({ type: e.WORKFLOW_COMPLETE, jobId, state: result }); // For tests diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index 1f9dbd2bb..71cce3ea3 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -7,7 +7,7 @@ // The sandbox should help // What about imports in a worker thread? -// Is there an overjhead in reimporting stuff (presumably!) +// Is there an overhead in reimporting stuff (presumably!) // Should we actually be pooling workers by adaptor[+version] // Does this increase the danger of sharing state between jobs? // Suddenly it's a liability for the same environent in the same adaptor @@ -15,13 +15,9 @@ import workerpool from 'workerpool'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; -import createLogger from '@openfn/logger'; -import helper from './worker-helper'; +import helper, { createLoggers } from './worker-helper'; -// TODO how can we control the logger in here? -// Need some kind of intitialisation function to set names and levels -const logger = createLogger('R/T', { level: 'debug' }); -const jobLogger = createLogger('JOB', { level: 'debug' }); +const { logger, jobLogger } = createLoggers(); workerpool.worker({ run: (plan: ExecutionPlan, repoDir: string) => { diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index 7b3a1ae75..02c88a76d 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -122,3 +122,28 @@ test('Publish workflow-complete event with state', async (t) => { t.true(didFire); t.deepEqual(state, { data: { answer: 42 } }); }); + +test('Publish a job log event', async (t) => { + const plan = createPlan({ + expression: `(s) => { + console.log('test') + return s; + }`, + }); + let didFire = false; + let log; + await workers.exec('run', [plan], { + on: ({ type, message }) => { + if (type === e.JOB_LOG) { + didFire = true; + log = message; + } + }, + }); + t.true(didFire); + + t.is(log.level, 'info'); + t.deepEqual(log.message, ['test']); + t.is(log.name, 'JOB'); + t.truthy(log.time); +}); From 47716ed96acc7d21256066da1560688fa3c05d39 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 15:05:43 +0100 Subject: [PATCH 026/232] rtm: refactor event names --- packages/runtime-manager/src/events.ts | 12 ++++++---- packages/runtime-manager/src/mock-worker.ts | 5 ++-- packages/runtime-manager/src/rtm.ts | 2 +- .../runtime-manager/src/runners/execute.ts | 6 ++--- packages/runtime-manager/src/worker-helper.ts | 24 +++++++++++-------- packages/runtime-manager/src/worker.ts | 4 ++-- .../runtime-manager/test/mock-worker.test.ts | 5 +++- 7 files changed, 33 insertions(+), 25 deletions(-) diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index fdfb4510c..51af201d2 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -1,3 +1,5 @@ +import { JSONLog } from '@openfn/logger'; + export const WORKFLOW_START = 'workflow-start'; export const WORKFLOW_COMPLETE = 'workflow-complete'; @@ -10,26 +12,26 @@ type State = any; // TODO export type AcceptWorkflowEvent = { type: typeof WORKFLOW_START; - jobId: string; + workflowId: string; threadId: number; }; export type CompleteWorkflowEvent = { type: typeof WORKFLOW_COMPLETE; - jobId: string; + workflowId: string; state: State; }; export type ErrWorkflowEvent = { type: typeof WORKFLOW_ERROR; - jobId: string; + workflowId: string; message: string; }; export type JobLogEvent = { type: typeof JOB_LOG; - jobId: string; - message: any; // TOD JSONLOG + workflowId: string; + message: JSONLog; }; export type WorkflowEvent = diff --git a/packages/runtime-manager/src/mock-worker.ts b/packages/runtime-manager/src/mock-worker.ts index aa3696e3b..e973573f5 100644 --- a/packages/runtime-manager/src/mock-worker.ts +++ b/packages/runtime-manager/src/mock-worker.ts @@ -10,8 +10,6 @@ import workerpool from 'workerpool'; import helper, { createLoggers } from './worker-helper'; -const { jobLogger } = createLoggers(); - type MockJob = { id?: string; adaptor?: string; @@ -33,6 +31,7 @@ type MockExecutionPlan = { // optionally delay function mock(plan: MockExecutionPlan) { const [job] = plan.jobs; + const { jobLogger } = createLoggers(plan.id!); return new Promise((resolve) => { setTimeout(async () => { // TODO this isn't data, but state - it's the whole state object (minus config) @@ -49,7 +48,7 @@ function mock(plan: MockExecutionPlan) { jobLogger ); state = await fn(state); - } catch (e) { + } catch (e: any) { state = { data: job.data || {}, error: { diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 49c31f5af..86db3f8bf 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -76,8 +76,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const onWorkflowStarted = (workflowId: string, threadId: number) => { logger.info('starting workflow ', workflowId); - const workflow = allWorkflows.get(workflowId)!; + if (workflow.startTime) { // TODO this shouldn't throw.. but what do we do? // We shouldn't run a workflow that's been run diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index 2cb9452f4..e63b37da4 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -21,10 +21,10 @@ const execute = ( .exec('run', [plan, repoDir], { on: ({ type, ...args }: e.WorkflowEvent) => { if (type === e.WORKFLOW_START) { - const { jobId, threadId } = args as e.AcceptWorkflowEvent; - start?.(jobId, threadId); + const { workflowId, threadId } = args as e.AcceptWorkflowEvent; + start?.(workflowId, threadId); } else if (type === e.WORKFLOW_COMPLETE) { - const { jobId, state } = args as e.CompleteWorkflowEvent; + const { workflowId, state } = args as e.CompleteWorkflowEvent; resolve(state); } }, diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index 7deb01d0f..d05605edd 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -3,7 +3,7 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; -import createLogger from '@openfn/logger'; +import createLogger, { JSONLog } from '@openfn/logger'; import * as e from './events'; @@ -11,12 +11,16 @@ function publish(event: e.WorkflowEvent) { workerpool.workerEmit(event); } -export const createLoggers = () => { - const log = (jsonLog: any) => { - publish({ type: e.JOB_LOG, message: JSON.parse(jsonLog) }); +export const createLoggers = (workflowId: string) => { + const log = (jsonLog: string) => { + publish({ + workflowId, + type: e.JOB_LOG, + message: JSON.parse(jsonLog) as JSONLog, + }); }; - const emitter = { + const emitter: any = { info: log, debug: log, log, @@ -40,21 +44,21 @@ export const createLoggers = () => { return { logger, jobLogger }; }; -async function helper(jobId: string, execute: () => Promise) { - publish({ type: e.WORKFLOW_START, jobId, threadId }); +async function helper(workflowId: string, execute: () => Promise) { + publish({ type: e.WORKFLOW_START, workflowId, threadId }); try { // Note that the worker thread may fire logs after completion - // I think this is fine, it's jsut a log stream thing + // I think this is fine, it's just a log stream thing // But the output is very confusing! const result = await execute(); - publish({ type: e.WORKFLOW_COMPLETE, jobId, state: result }); + publish({ type: e.WORKFLOW_COMPLETE, workflowId, state: result }); // For tests return result; } catch (err) { console.error(err); // @ts-ignore TODO sort out error typing - publish({ type: e.WORKFLOW_ERROR, jobId, message: err.message }); + publish({ type: e.WORKFLOW_ERROR, workflowId, message: err.message }); } } diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index 71cce3ea3..b91fa2ef3 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -17,10 +17,10 @@ import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; import helper, { createLoggers } from './worker-helper'; -const { logger, jobLogger } = createLoggers(); - workerpool.worker({ run: (plan: ExecutionPlan, repoDir: string) => { + const { logger, jobLogger } = createLoggers(plan.id!); + const options = { logger, jobLogger, diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index 02c88a76d..81bc2fcad 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -132,15 +132,18 @@ test('Publish a job log event', async (t) => { }); let didFire = false; let log; + let id; await workers.exec('run', [plan], { - on: ({ type, message }) => { + on: ({ workflowId, type, message }) => { if (type === e.JOB_LOG) { didFire = true; log = message; + id = workflowId; } }, }); t.true(didFire); + t.is(id, plan.id); t.is(log.level, 'info'); t.deepEqual(log.message, ['test']); From 2c76967df3b62f9a4f0126dcf6cb1a781257149c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 15:07:45 +0100 Subject: [PATCH 027/232] rtm: job-log -> workflow-log --- packages/runtime-manager/src/events.ts | 8 ++++---- packages/runtime-manager/src/worker-helper.ts | 2 +- .../runtime-manager/test/mock-worker.test.ts | 2 +- packages/runtime-manager/test/rtm.test.ts | 19 +++++++++++++++++++ 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index 51af201d2..531db3c21 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -6,7 +6,7 @@ export const WORKFLOW_COMPLETE = 'workflow-complete'; export const WORKFLOW_ERROR = 'workflow-error'; -export const JOB_LOG = 'job-log'; +export const WORKFLOW_LOG = 'workflow-log'; type State = any; // TODO @@ -28,8 +28,8 @@ export type ErrWorkflowEvent = { message: string; }; -export type JobLogEvent = { - type: typeof JOB_LOG; +export type WorkflowLogEvent = { + type: typeof WORKFLOW_LOG; workflowId: string; message: JSONLog; }; @@ -38,4 +38,4 @@ export type WorkflowEvent = | AcceptWorkflowEvent | CompleteWorkflowEvent | ErrWorkflowEvent - | JobLogEvent; + | WorkflowLogEvent; diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/runtime-manager/src/worker-helper.ts index d05605edd..d5f032639 100644 --- a/packages/runtime-manager/src/worker-helper.ts +++ b/packages/runtime-manager/src/worker-helper.ts @@ -15,7 +15,7 @@ export const createLoggers = (workflowId: string) => { const log = (jsonLog: string) => { publish({ workflowId, - type: e.JOB_LOG, + type: e.WORKFLOW_LOG, message: JSON.parse(jsonLog) as JSONLog, }); }; diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/runtime-manager/test/mock-worker.test.ts index 81bc2fcad..f39db8b75 100644 --- a/packages/runtime-manager/test/mock-worker.test.ts +++ b/packages/runtime-manager/test/mock-worker.test.ts @@ -135,7 +135,7 @@ test('Publish a job log event', async (t) => { let id; await workers.exec('run', [plan], { on: ({ workflowId, type, message }) => { - if (type === e.JOB_LOG) { + if (type === e.WORKFLOW_LOG) { didFire = true; log = message; id = workflowId; diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts index e4cb18dd2..43aabd80e 100644 --- a/packages/runtime-manager/test/rtm.test.ts +++ b/packages/runtime-manager/test/rtm.test.ts @@ -90,4 +90,23 @@ test('events: workflow-complete', async (t) => { t.deepEqual(evt.state, { data: { answer: 42 } }); }); +test('events: workflow-log', async (t) => { + const rtm = Manager('x', options); + + let didCall; + let evt; + rtm.on(e.WORKFLOW_COMPLETE, (e) => { + didCall = true; + evt = e; + }); + + const plan = createPlan(); + await rtm.execute(plan); + + t.true(didCall); + t.is(evt.workflowId, plan.id); + t.truthy(evt.duration); + t.deepEqual(evt.state, { data: { answer: 42 } }); +}); + // TODO events: logging. How will I test this with the mock? From 2dfdc4ebee3e9c900b5024e2ab4cc962f26a08bf Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 16:28:11 +0100 Subject: [PATCH 028/232] logger: add proxy function --- packages/logger/src/logger.ts | 37 +++++++++++++++++---- packages/logger/test/logger.test.ts | 50 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 6 deletions(-) diff --git a/packages/logger/src/logger.ts b/packages/logger/src/logger.ts index fdc3e3843..2c9b884d8 100644 --- a/packages/logger/src/logger.ts +++ b/packages/logger/src/logger.ts @@ -72,6 +72,8 @@ export interface Logger extends Console { always(...args: any[]): void; // fancier log functions + proxy(obj: Partial): void; + proxy(name: string, level: string, message: any[]): void; print(...args: any[]): void; confirm(message: string, force?: boolean): Promise; timer(name: string): string | undefined; @@ -141,17 +143,21 @@ export default function (name?: string, options: LogOptions = {}): Logger { // This is what we actually pass the log strings to const emitter = opts.logger; - const log = (level: LogFns, ...args: LogArgs) => { + const log = (name: string | undefined, level: LogFns, ...args: LogArgs) => { if (priority[level] >= minLevel) { if (options.json) { - logJSON(level, ...args); + logJSON(name, level, ...args); } else { - logString(level, ...args); + logString(name, level, ...args); } } }; - const logJSON = (level: LogFns, ...args: LogArgs) => { + const logJSON = ( + name: string | undefined, + level: LogFns, + ...args: LogArgs + ) => { const output: JSONLog = { level, name, @@ -163,7 +169,11 @@ export default function (name?: string, options: LogOptions = {}): Logger { emitter[level](stringify(output)); }; - const logString = (level: LogFns, ...args: LogArgs) => { + const logString = ( + name: string | undefined, + level: LogFns, + ...args: LogArgs + ) => { if (emitter.hasOwnProperty(level)) { const output = []; @@ -183,6 +193,20 @@ export default function (name?: string, options: LogOptions = {}): Logger { } }; + // "forward" a log event from another logger as if it came from this one + const proxy = function (...args: any[]) { + let j; + if (args.length === 3) { + const [name, level, message] = args; + j = { name, level, message }; + } else { + j = args[0]; + } + j = j as JSONLog; + + log(j.name, j.level, ...j.message); + }; + // print() will log without any metadata/overhead/santization // basically a proxy for console.log const print = (...args: any[]) => { @@ -220,7 +244,7 @@ export default function (name?: string, options: LogOptions = {}): Logger { const wrap = (level: LogFns) => (...args: LogArgs) => - log(level, ...args); + log(name, level, ...args); // TODO this does not yet cover the full console API const logger = { @@ -234,6 +258,7 @@ export default function (name?: string, options: LogOptions = {}): Logger { confirm, timer, print, + proxy, // possible convenience APIs force: () => {}, // force the next lines to log (even if silent) diff --git a/packages/logger/test/logger.test.ts b/packages/logger/test/logger.test.ts index ab5c989d4..b6dad782c 100644 --- a/packages/logger/test/logger.test.ts +++ b/packages/logger/test/logger.test.ts @@ -402,3 +402,53 @@ test('log an error object', (t) => { const { message } = logger._parse(logger._last); t.assert(message instanceof Error); }); + +test('proxy a json argument to string', (t) => { + const logger = createLogger('x'); + logger.proxy({ name: 'y', level: 'success', message: ['hello'] }); + + const { namespace, level, message } = logger._parse(logger._last); + t.is(namespace, 'y'); + t.is(level, 'success'); + t.deepEqual(message, 'hello'); +}); + +test('proxy string arguments to string', (t) => { + const logger = createLogger('x'); + logger.proxy('y', 'success', ['hello']); + + const { namespace, level, message } = logger._parse(logger._last); + t.is(namespace, 'y'); + t.is(level, 'success'); + t.deepEqual(message, 'hello'); +}); + +test('proxy a json argument to json', (t) => { + const logger = createLogger('x', { json: true }); + logger.proxy({ name: 'y', level: 'success', message: ['hello'] }); + + const { name, level, message } = JSON.parse(logger._last as any); + t.is(name, 'y'); + t.is(level, 'success'); + t.deepEqual(message, ['hello']); +}); + +test('proxy string arguments to json', (t) => { + const logger = createLogger('x', { json: true }); + logger.proxy('y', 'success', ['hello']); + + const { name, level, message } = JSON.parse(logger._last as any); + t.is(name, 'y'); + t.is(level, 'success'); + t.deepEqual(message, ['hello']); +}); + +test('proxy should respect log levels', (t) => { + const logger = createLogger('x', { level: 'default' }); + logger.proxy({ level: 'debug', name: '', message: ['hidden'] }); + + // do nothing + + const [last] = logger._last; + t.falsy(last); +}); From 1b6fa8ee0739c3a03b80b307d073463b2c28225b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 16:28:43 +0100 Subject: [PATCH 029/232] logger: changeset --- .changeset/warm-tables-explode.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/warm-tables-explode.md diff --git a/.changeset/warm-tables-explode.md b/.changeset/warm-tables-explode.md new file mode 100644 index 000000000..c4c215802 --- /dev/null +++ b/.changeset/warm-tables-explode.md @@ -0,0 +1,5 @@ +--- +'@openfn/logger': patch +--- + +Add proxy function" From 85b677b040b17907c9d111759da5ae3d09c7e6e5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 16:31:48 +0100 Subject: [PATCH 030/232] rtm: log and forward runtime and job logs --- packages/runtime-manager/src/events.ts | 4 ++-- packages/runtime-manager/src/rtm.ts | 18 ++++++++++++++- .../runtime-manager/src/runners/execute.ts | 3 +++ packages/runtime-manager/test/rtm.test.ts | 23 +++++++++++-------- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/packages/runtime-manager/src/events.ts b/packages/runtime-manager/src/events.ts index 531db3c21..0daf791f0 100644 --- a/packages/runtime-manager/src/events.ts +++ b/packages/runtime-manager/src/events.ts @@ -28,7 +28,7 @@ export type ErrWorkflowEvent = { message: string; }; -export type WorkflowLogEvent = { +export type LogWorkflowEvent = { type: typeof WORKFLOW_LOG; workflowId: string; message: JSONLog; @@ -38,4 +38,4 @@ export type WorkflowEvent = | AcceptWorkflowEvent | CompleteWorkflowEvent | ErrWorkflowEvent - | WorkflowLogEvent; + | LogWorkflowEvent; diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 86db3f8bf..f26e1753d 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -9,7 +9,7 @@ import * as e from './events'; // import createAutoinstall from './runners/autoinstall'; import createCompile from './runners/compile'; import createExecute from './runners/execute'; -import createLogger, { Logger } from '@openfn/logger'; +import createLogger, { JSONLog, Logger } from '@openfn/logger'; export type State = any; // TODO I want a nice state def with generics @@ -52,6 +52,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const id = serverId || crypto.randomUUID(); const logger = options.logger || createLogger('RTM', { level: 'debug' }); + const runtimeLogger = createLogger('R/T', { level: 'debug' }); + const allWorkflows: Map = new Map(); const activeWorkflows: string[] = []; @@ -118,9 +120,23 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { }); }; + // Catch a log coming out of a job within a workflow + // Includes runtime logging (is this right?) + const onWorkflowLog = (workflowId: string, message: JSONLog) => { + // Seamlessly proxy the log to the local stdout + // TODO runtime logging probably needs to be at info level? + // Debug information is mostly irrelevant for lightning + logger.proxy(message); + events.emit(e.WORKFLOW_LOG, { + workflowId, + message, + }); + }; + // Create "runner" functions for execute and compile const execute = createExecute(workers, repoDir, logger, { start: onWorkflowStarted, + log: onWorkflowLog, }); const compile = createCompile(logger, repoDir); diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index e63b37da4..d7154d74b 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -26,6 +26,9 @@ const execute = ( } else if (type === e.WORKFLOW_COMPLETE) { const { workflowId, state } = args as e.CompleteWorkflowEvent; resolve(state); + } else if (type === e.WORKFLOW_LOG) { + const { workflowId, message } = args as e.LogWorkflowEvent; + log(workflowId, message); } }, }) diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts index 43aabd80e..a9095483f 100644 --- a/packages/runtime-manager/test/rtm.test.ts +++ b/packages/runtime-manager/test/rtm.test.ts @@ -90,23 +90,28 @@ test('events: workflow-complete', async (t) => { t.deepEqual(evt.state, { data: { answer: 42 } }); }); -test('events: workflow-log', async (t) => { +// TODO: workflow log should also include runtime events, which maybe should be reflected here + +test('events: workflow-log from a job', async (t) => { const rtm = Manager('x', options); let didCall; let evt; - rtm.on(e.WORKFLOW_COMPLETE, (e) => { + rtm.on(e.WORKFLOW_LOG, (e) => { didCall = true; evt = e; }); - const plan = createPlan(); + const plan = createPlan({ + expression: `(s) => { + console.log('log me') + return s; + }`, + }); await rtm.execute(plan); - t.true(didCall); - t.is(evt.workflowId, plan.id); - t.truthy(evt.duration); - t.deepEqual(evt.state, { data: { answer: 42 } }); -}); -// TODO events: logging. How will I test this with the mock? + t.is(evt.message.level, 'info'); + t.deepEqual(evt.message.message, ['log me']); + t.is(evt.message.name, 'JOB'); +}); From 0ef907b94eb722a9f8acc2a3f47b1423a029dadb Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 16:38:52 +0100 Subject: [PATCH 031/232] Remove old stuff --- packages/runtime-manager/package.json | 3 +- .../src/{server => }/jobs/slow-random.js | 4 +- packages/runtime-manager/src/server/index.ts | 78 ------------------- .../test/jobs/slow-random.test.ts | 2 +- packages/runtime-manager/test/rtm.test.ts | 2 +- 5 files changed, 6 insertions(+), 83 deletions(-) rename packages/runtime-manager/src/{server => }/jobs/slow-random.js (79%) delete mode 100644 packages/runtime-manager/src/server/index.ts diff --git a/packages/runtime-manager/package.json b/packages/runtime-manager/package.json index 6cf3b0ab7..50ace3b70 100644 --- a/packages/runtime-manager/package.json +++ b/packages/runtime-manager/package.json @@ -9,8 +9,7 @@ "test": "pnpm ava", "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", - "build:watch": "pnpm build --watch", - "serve": "nodemon -e ts --exec 'tsm --experimental-vm-modules --no-warnings src/server/index.ts'" + "build:watch": "pnpm build --watch" }, "author": "Open Function Group ", "license": "ISC", diff --git a/packages/runtime-manager/src/server/jobs/slow-random.js b/packages/runtime-manager/src/jobs/slow-random.js similarity index 79% rename from packages/runtime-manager/src/server/jobs/slow-random.js rename to packages/runtime-manager/src/jobs/slow-random.js index f2106cb93..1f9958621 100644 --- a/packages/runtime-manager/src/server/jobs/slow-random.js +++ b/packages/runtime-manager/src/jobs/slow-random.js @@ -1,8 +1,10 @@ +// TODO this is old code but it may still have a use - keeping it around for now + // This job takes a random number of seconds and returns a random number // import { fn } from '@openfn/language-common'; // Note: since the linker no longer uses node resolution for import, common is unavailable -// to the run-time manager. We basically need to add repo support. +// to the runtime manager. We basically need to add repo support. // For now, this simulates the behaviour well enough const fn = (f) => f; diff --git a/packages/runtime-manager/src/server/index.ts b/packages/runtime-manager/src/server/index.ts deleted file mode 100644 index b27696466..000000000 --- a/packages/runtime-manager/src/server/index.ts +++ /dev/null @@ -1,78 +0,0 @@ -import koa from 'koa'; -import fs from 'node:fs/promises'; -import path from 'node:path'; -import Manager from '../rtm'; - -const loadJobs = async () => { - for (const name of ['slow-random']) { - const source = await fs.readFile( - path.resolve(`src/server/jobs/${name}.js`), - { encoding: 'utf8' } - ); - runtime.registerJob(name, source); - } - console.log('Jobs loaded:'); - console.log(runtime.getRegisteredJobs()); -}; - -const app = new koa(); - -console.log('starting server'); - -const runtime = Manager(); - -loadJobs(); - -// Create http server -// GET works return alist of workers - -// on post to job/name we run that job - -// need a web socket to listen to and report changes - -const handlePost = (ctx: koa.Context) => { - ctx; - const state = { - configuration: { - delay: Math.random() * 10000, - }, - }; - // start a job - runJob('slow-random', state); -}; - -const runJob = async (name: string, state: any) => { - console.log(`Starting job: ${name}...`); - - const result = await runtime.run(name, state); - - // console.log('--') - console.log(`Job ${name} finished in ${result.duration / 1000}s`); - console.log(result.result); - // console.log('--') - report(); -}; - -const report = () => { - const jobs = runtime.getActiveJobs(); - const oldJobs = runtime.getCompletedJobs(); - console.log('---'); - console.log(`completed jobs: ${oldJobs.length}`); - console.log(`active jobs (${jobs.length}):`); - for (const job of jobs) { - console.log(` [${job.id}] ${job.name}: (thread: ${job.threadId})`); - } - console.log('---'); -}; - -app.use((ctx) => { - if (ctx.method === 'POST') { - handlePost(ctx); - } -}); - -app.listen(1234); - -report(); - -export default {}; diff --git a/packages/runtime-manager/test/jobs/slow-random.test.ts b/packages/runtime-manager/test/jobs/slow-random.test.ts index 393d6faf3..67ad7827a 100644 --- a/packages/runtime-manager/test/jobs/slow-random.test.ts +++ b/packages/runtime-manager/test/jobs/slow-random.test.ts @@ -13,7 +13,7 @@ const wait = async (time: number) => setTimeout(resolve, time); }); -const compiledJob = compile('src/server/jobs/slow-random.js'); +const compiledJob = compile('src/jobs/slow-random.js'); test('slowmo should return a value', async (t) => { const result = (await execute(compiledJob)) as SlowMoState; diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts index a9095483f..83929337a 100644 --- a/packages/runtime-manager/test/rtm.test.ts +++ b/packages/runtime-manager/test/rtm.test.ts @@ -86,7 +86,7 @@ test('events: workflow-complete', async (t) => { t.true(didCall); t.is(evt.workflowId, plan.id); - t.truthy(evt.duration); + t.assert(!isNaN(evt.duration)); t.deepEqual(evt.state, { data: { answer: 42 } }); }); From f0786c4520d5c104ae581196d12699007d4b65b2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 6 Jun 2023 17:38:11 +0100 Subject: [PATCH 032/232] rtm: typings --- packages/runtime-manager/src/rtm.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index f26e1753d..851e42535 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -42,18 +42,16 @@ type RTMOptions = { logger?: Logger; workerPath?: string; repoDir?: string; - noCompile: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? + noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? }; const createRTM = function (serverId?: string, options: RTMOptions = {}) { - const { resolvers, noCompile } = options; + const { noCompile } = options; let { repoDir, workerPath } = options; const id = serverId || crypto.randomUUID(); const logger = options.logger || createLogger('RTM', { level: 'debug' }); - const runtimeLogger = createLogger('R/T', { level: 'debug' }); - const allWorkflows: Map = new Map(); const activeWorkflows: string[] = []; @@ -108,7 +106,7 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const workflow = allWorkflows.get(workflowId)!; workflow.status = 'done'; workflow.result = state; - workflow.duration = new Date().getTime() - workflow.startTime; + workflow.duration = new Date().getTime() - workflow.startTime!; const idx = activeWorkflows.findIndex((id) => id === workflowId); activeWorkflows.splice(idx, 1); From 1bf67a0e22f2b27136285cd31146861c4a3bcd5e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 7 Jun 2023 11:30:00 +0100 Subject: [PATCH 033/232] rtm-server: fix index,remove comment --- packages/rtm-server/src/index.ts | 4 +--- packages/rtm-server/src/start.ts | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/rtm-server/src/index.ts b/packages/rtm-server/src/index.ts index 2e5d5b278..1199af15d 100644 --- a/packages/rtm-server/src/index.ts +++ b/packages/rtm-server/src/index.ts @@ -1,3 +1 @@ -import createServer from './server'; - -const server = createServer(); +import './server'; diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index 9ab177f93..72e5c9b28 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -31,8 +31,6 @@ const args = yargs(hideBin(process.argv)) const rtm = createRTM(); logger.debug('RTM created'); -// TODO why is this blowing up?? - createRTMServer(rtm, { port: args.port, lightning: args.lightning, From 9cf78f6efcc3e8b574f36a486153ad3279d9cadf Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 8 Jun 2023 10:21:20 +0100 Subject: [PATCH 034/232] rtm-server: flesh out lightning mock a bit --- packages/rtm-server/package.json | 1 + packages/rtm-server/src/mock/lightning/api.ts | 31 ++++++++++--------- .../src/mock/lightning/middleware.ts | 5 +++ .../rtm-server/src/mock/lightning/server.ts | 7 ++++- .../rtm-server/src/mock/lightning/start.ts | 8 +++++ 5 files changed, 36 insertions(+), 16 deletions(-) create mode 100644 packages/rtm-server/src/mock/lightning/start.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index d7185722d..d7764bc9a 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -11,6 +11,7 @@ "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm src/start.ts", + "start:lightning": "ts-node-esm --transpile-only src/mock/lightning/start.ts", "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'" }, "author": "Open Function Group ", diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index e158394f4..3b19b37ab 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -1,6 +1,7 @@ import type Router from '@koa/router'; import { unimplemented, + createListNextJob, createFetchNextJob, createGetCredential, createLog, @@ -8,8 +9,6 @@ import { } from './middleware'; import type { ServerState } from './server'; -export const API_PREFIX = `/api/1`; - interface RTMBody { rtm_id: string; } @@ -20,7 +19,9 @@ export interface AttemptCompleteBody extends RTMBody { state: any; // JSON state object (undefined? null?) } -export default (router: Router, state: ServerState) => { +export default (router: Router, logger, state: ServerState) => { + router.use(logger); + // Basically all requests must include an rtm_id // And probably later a security token @@ -29,39 +30,39 @@ export default (router: Router, state: ServerState) => { // Lightning should track who has each attempt // 200 - return an array of pending attempts // 204 - queue empty (no body) - router.post(`${API_PREFIX}/attempts/next`, createFetchNextJob(state)); + router.post('/attempts/next', createFetchNextJob(state)); // GET credential/:id // Get a credential // 200 - return a credential object // 404 - credential not found - router.get(`${API_PREFIX}/credential/:id`, createGetCredential(state)); + router.get('/credential/:id', createGetCredential(state)); // Notify for a batch of job logs // [{ rtm_id, logs: ['hello world' ] }] // TODO this could use a websocket to handle the high volume of logs - router.post(`${API_PREFIX}/attempts/log/:id`, createLog(state)); + router.post('/attempts/log/:id', createLog(state)); // Notify an attempt has finished // Could be error or success state // If a complete comes in from an unexpected source (ie a timed out job), this should throw // state and rtm_id should be in the payload // { rtm,_id, state } | { rtmId, error } - router.post(`${API_PREFIX}/attempts/complete/:id`, createComplete(state)); + router.post('/attempts/complete/:id', createComplete(state)); // TODO i want this too: confirm that an attempt has started - router.post(`${API_PREFIX}/attempts/start/:id`, () => {}); + router.post('/attempts/start/:id', () => {}); // Listing APIs - these list details without changing anything - router.get(`${API_PREFIX}/attempts/:id`, unimplemented); - router.get(`${API_PREFIX}/attempts/next`, unimplemented); // ?count=1 - router.get(`${API_PREFIX}/attempts/done`, unimplemented); // ?project=pid - router.get(`${API_PREFIX}/attempts/active`, unimplemented); + router.get('/attempts/next', createListNextJob(state)); // ?count=1 + router.get('/attempts/:id', unimplemented); + router.get('/attempts/done', unimplemented); // ?project=pid + router.get('/attempts/active', unimplemented); - router.get(`${API_PREFIX}/credential/:id`, unimplemented); + router.get('/credential/:id', unimplemented); - router.get(`${API_PREFIX}/workflows`, unimplemented); - router.get(`${API_PREFIX}/workflows/:id`, unimplemented); + router.get('/workflows', unimplemented); + router.get('/workflows/:id', unimplemented); return router; }; diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 9ce699a17..9d6b50997 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -53,6 +53,11 @@ export const createFetchNextJob = } }; +export const createListNextJob = (state: ServerState) => (ctx: Koa.Context) => { + ctx.body = state.queue.map(({ id }) => id); + ctx.status = 200; +}; + export const createGetCredential = (state: ServerState) => (ctx: Koa.Context) => { const { credentials } = state; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 66b17f9f1..f9f4d3aad 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -1,6 +1,7 @@ import { EventEmitter } from 'node:events'; import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; +import koaLogger from 'koa-logger'; import Router from '@koa/router'; import createAPI from './api'; @@ -37,12 +38,16 @@ const createLightningServer = (options = {}) => { // Server setup const app = new Koa(); - const api = createAPI(new Router(), state); app.use(bodyParser()); + const logger = koaLogger(); + // Mock API endpoints + const api = createAPI(new Router({ prefix: '/api/1' }), logger, state); app.use(api.routes()); + app.use(logger); + // Dev APIs for unit testing app.addCredential = (id: string, cred: Credential) => { state.credentials[id] = cred; diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts new file mode 100644 index 000000000..d8f3a1663 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -0,0 +1,8 @@ +import createLightningServer from './server'; + +const port = 8888; + +createLightningServer({ + port, +}); +console.log('Starting mock Lightning server on ', port); From c036e7d5e16ca2f2e2b7f25493bdbccbc219e735 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 8 Jun 2023 16:02:40 +0100 Subject: [PATCH 035/232] rtm-server: refactor dev apis for lightning, add some docs --- packages/rtm-server/README.md | 23 ++++ .../rtm-server/src/mock/lightning/api-dev.ts | 104 ++++++++++++++++++ .../rtm-server/src/mock/lightning/index.ts | 2 +- .../src/mock/lightning/middleware.ts | 2 +- .../rtm-server/src/mock/lightning/server.ts | 79 +++---------- .../rtm-server/src/mock/lightning/start.ts | 3 +- .../rtm-server/test/mock/lightning.test.ts | 85 +++++++------- 7 files changed, 190 insertions(+), 108 deletions(-) create mode 100644 packages/rtm-server/README.md create mode 100644 packages/rtm-server/src/mock/lightning/api-dev.ts diff --git a/packages/rtm-server/README.md b/packages/rtm-server/README.md new file mode 100644 index 000000000..f84e51141 --- /dev/null +++ b/packages/rtm-server/README.md @@ -0,0 +1,23 @@ +## Lightning Mock + +You can start a Lightning mock server with: +``` +pnpm start:lightning +``` + +This will run on port 8888 [TODO: drop yargs in to customise the port] + +Get the Attempts queue with: +``` +curl http://localhost:8888/api/1/attempts/next +``` +Add an attempt (`{ jobs, triggers, edges }`) to the queue with: +``` +curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" +``` +Get an attempt with +``` +curl http://localhost:8888/api/1/attempts/next/:id +``` + + diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts new file mode 100644 index 000000000..5b140da9a --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -0,0 +1,104 @@ +/* + * This module sets up a bunch of dev-only APIs + * These are not intended to be reflected in Lightning itself + */ +import Koa from 'koa'; +import Router from '@koa/router'; +import { Logger } from '@openfn/logger'; +import crypto from 'node:crypto'; + +import { Attempt } from '../../types'; +import { ServerState } from './server'; + +type LightningEvents = 'log' | 'attempt-complete'; + +export type DevApp = Koa & { + addCredential(id: string, cred: Credential): void; + waitForResult(attemptId: string): Promise; + enqueueAttempt(attempt: Attempt, rtmId: string): void; + reset(): void; + getQueueLength(): number; + getResult(attemptId: string): any; + on(event: LightningEvents, fn: (evt: any) => void): void; + once(event: LightningEvents, fn: (evt: any) => void): void; +}; + +const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger) => { + // Dev APIs for unit testing + app.addCredential = (id: string, cred: Credential) => { + logger.info(`Add credential ${id}`); + state.credentials[id] = cred; + }; + + // Promise which returns when a workflow is complete + app.waitForResult = (attemptId: string) => { + return new Promise((resolve) => { + const handler = (evt: any) => { + if (evt.workflow_id === attemptId) { + state.events.removeListener('attempt-complete', handler); + resolve(evt); + } + }; + state.events.addListener('attempt-complete', handler); + }); + }; + + // Add an attempt to the queue + // TODO actually it shouldn't take an rtm id until it's pulled off the attempt + // Something feels off here + app.enqueueAttempt = (attempt: Attempt, rtmId: string = 'rtm') => { + logger.info(`Add Attempt ${attempt.id}`); + + state.results[attempt.id] = { + rtmId, + state: null, + }; + state.queue.push(attempt); + }; + + app.reset = () => { + state.queue = []; + state.results = {}; + }; + + app.getQueueLength = () => state.queue.length; + + app.getResult = (attemptId: string) => state.results[attemptId]?.state; + + // TODO these are overriding koa's event handler - should I be doing something different? + + // @ts-ignore + app.on = (event: LightningEvents, fn: (evt: any) => void) => { + state.events.addListener(event, fn); + }; + + // @ts-ignore + app.once = (event: LightningEvents, fn: (evt: any) => void) => { + state.events.once(event, fn); + }; +}; + +// Set up some rest endpoints +// Note that these are NOT prefixed +const setupRestAPI = (app: DevApp, state: ServerState, logger) => { + const router = new Router(); + + router.post('/attempt', (ctx) => { + const data = ctx.request.body; + const rtmId = 'rtm'; // TODO include this in the body maybe? + if (!data.id) { + data.id = crypto.randomUUID(); + logger.info('Generating new id for incoming attempt:', data.id); + } + app.enqueueAttempt(data, rtmId); + + ctx.response.status = 200; + }); + + app.use(router.routes()); +}; + +export default (app: DevApp, state: ServerState, logger: Logger) => { + setupDevAPI(app, state, logger); + setupRestAPI(app, state, logger); +}; diff --git a/packages/rtm-server/src/mock/lightning/index.ts b/packages/rtm-server/src/mock/lightning/index.ts index dc631ee27..b205885b6 100644 --- a/packages/rtm-server/src/mock/lightning/index.ts +++ b/packages/rtm-server/src/mock/lightning/index.ts @@ -1,4 +1,4 @@ import createLightningServer from './server'; export default createLightningServer; -export { API_PREFIX } from './api'; +export { API_PREFIX } from './server'; diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 9d6b50997..1f4e2a272 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -102,7 +102,7 @@ export const createComplete = if (shouldAcceptRequest(state, ctx.params.id, ctx.request)) { results[ctx.params.id].state = resultState; - events.emit('workflow-complete', { + events.emit('attempt-complete', { rtm_id, workflow_id: ctx.params.id, state: resultState, diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index f9f4d3aad..c3c21e725 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -3,16 +3,13 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import Router from '@koa/router'; +import createLogger, { LogLevel, Logger } from '@openfn/logger'; import createAPI from './api'; +import createDevAPI from './api-dev'; import { Attempt } from '../../types'; -import { RTMEvent } from '../runtime-manager'; -type NotifyEvent = { - event: RTMEvent; - workflow: string; // workflow id - [key: string]: any; -}; +export const API_PREFIX = '/api/1'; export type ServerState = { credentials: Record; @@ -22,8 +19,17 @@ export type ServerState = { events: EventEmitter; }; +export type LightningOptions = { + logger?: Logger; + logLevel?: LogLevel; + port?: string; +}; + // a mock lightning server -const createLightningServer = (options = {}) => { +const createLightningServer = (options: LightningOptions = {}) => { + const logger = + options.logger || + createLogger('LNG', { level: options.logLevel || 'info' }); // App state const state = { credentials: {}, @@ -40,67 +46,16 @@ const createLightningServer = (options = {}) => { const app = new Koa(); app.use(bodyParser()); - const logger = koaLogger(); + const klogger = koaLogger((str) => logger.log(str)); + app.use(klogger); // Mock API endpoints - const api = createAPI(new Router({ prefix: '/api/1' }), logger, state); + const api = createAPI(new Router({ prefix: API_PREFIX }), klogger, state); app.use(api.routes()); - app.use(logger); - - // Dev APIs for unit testing - app.addCredential = (id: string, cred: Credential) => { - state.credentials[id] = cred; - }; - app.addAttempt = (attempt: Attempt) => { - state.attempts[attempt.id] = attempt; - }; - app.addToQueue = (attempt: string | Attempt, rtmId: string = 'rtm') => { - if (typeof attempt == 'string') { - app.addPendingWorkflow(attempt, rtmId); - if (state.attempts[attempt]) { - state.queue.push(state.attempts[attempt]); - return true; - } - throw new Error(`attempt ${attempt} not found`); - } else if (attempt) { - app.addPendingWorkflow(attempt.id, rtmId); - state.queue.push(attempt); - return true; - } - }; - app.waitForResult = (workflowId: string) => { - return new Promise((resolve) => { - const handler = (evt) => { - if (evt.workflow_id === workflowId) { - state.events.removeListener('workflow-complete', handler); - resolve(evt); - } - }; - state.events.addListener('workflow-complete', handler); - }); - }; - app.addPendingWorkflow = (workflowId: string, rtmId: string) => { - state.results[workflowId] = { - rtmId, - state: null, - }; - }; - app.reset = () => { - state.queue = []; - state.results = {}; - }; - app.getQueueLength = () => state.queue.length; - app.getResult = (attemptId: string) => state.results[attemptId]?.state; - app.on = (event: 'notify', fn: (evt: any) => void) => { - state.events.addListener(event, fn); - }; - app.once = (event: 'notify', fn: (evt: any) => void) => { - state.events.once(event, fn); - }; + createDevAPI(app, state, logger); const server = app.listen(options.port || 8888); - app.destroy = () => { server.close(); }; diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts index d8f3a1663..a53ba3d5d 100644 --- a/packages/rtm-server/src/mock/lightning/start.ts +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -5,4 +5,5 @@ const port = 8888; createLightningServer({ port, }); -console.log('Starting mock Lightning server on ', port); + +console.log('Started mock Lightning server on ', port); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 05c25a139..d0fcf497b 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,13 +1,15 @@ import test from 'ava'; import { attempts } from '../../src/mock/data'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; +import { createMockLogger } from '@openfn/logger'; const baseUrl = `http://localhost:8888${API_PREFIX}`; let server; test.before(() => { - server = createLightningServer({ port: 8888 }); + const logger = createMockLogger(); + server = createLightningServer({ port: 8888, logger }); }); test.afterEach(() => { @@ -67,7 +69,7 @@ test.serial('POST /attempts/next - return 400 if no id provided', async (t) => { }); test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { - server.addToQueue(attempt1); + server.enqueueAttempt(attempt1); t.is(server.getQueueLength(), 1); const res = await post('attempts/next', { rtm_id: 'rtm' }); @@ -89,7 +91,7 @@ test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { test.serial( 'GET /attempts/next - return 200 with a workflow with an inline item', async (t) => { - server.addToQueue({ id: 'abc' }); + server.enqueueAttempt({ id: 'abc' }); t.is(server.getQueueLength(), 1); const res = await post('attempts/next', { rtm_id: 'rtm' }); @@ -108,9 +110,9 @@ test.serial( ); test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { - server.addToQueue(attempt1); - server.addToQueue(attempt1); - server.addToQueue(attempt1); + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); t.is(server.getQueueLength(), 3); const res = await post('attempts/next?count=2', { rtm_id: 'rtm' }); @@ -127,7 +129,7 @@ test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { test.serial( 'POST /attempts/next - clear the queue after a request', async (t) => { - server.addToQueue(attempt1); + server.enqueueAttempt(attempt1); const res1 = await post('attempts/next', { rtm_id: 'rtm' }); t.is(res1.status, 200); @@ -140,8 +142,8 @@ test.serial( ); test.serial('POST /attempts/log - should return 200', async (t) => { - server.addPendingWorkflow('a', 'rtm'); - const { status } = await post('attempts/log/a', { + server.enqueueAttempt(attempt1); + const { status } = await post('attempts/log/attempt-1', { rtm_id: 'rtm', logs: [{ message: 'hello world' }], }); @@ -151,7 +153,7 @@ test.serial('POST /attempts/log - should return 200', async (t) => { test.serial( 'POST /attempts/log - should return 400 if no rtm_id', async (t) => { - const { status } = await post('attempts/log/a', { + const { status } = await post('attempts/log/attempt-1', { rtm_id: 'rtm', logs: [{ message: 'hello world' }], }); @@ -159,61 +161,58 @@ test.serial( } ); -test.serial( - 'POST /attempts/notify - should echo to event emitter', - async (t) => { - server.addPendingWorkflow('a', 'rtm'); - let evt; - let didCall = false; +test.serial('POST /attempts/log - should echo to event emitter', async (t) => { + server.enqueueAttempt(attempt1); + let evt; + let didCall = false; - server.once('log', (e) => { - didCall = true; - evt = e; - }); + server.once('log', (e) => { + didCall = true; + evt = e; + }); - const { status } = await post('attempts/log/a', { - rtm_id: 'rtm', - logs: [{ message: 'hello world' }], - }); - t.is(status, 200); - t.true(didCall); + const { status } = await post('attempts/log/attempt-1', { + rtm_id: 'rtm', + logs: [{ message: 'hello world' }], + }); + t.is(status, 200); + t.true(didCall); - t.truthy(evt); - t.is(evt.id, 'a'); - t.deepEqual(evt.logs, [{ message: 'hello world' }]); - } -); + t.truthy(evt); + t.is(evt.id, 'attempt-1'); + t.deepEqual(evt.logs, [{ message: 'hello world' }]); +}); test.serial('POST /attempts/complete - return final state', async (t) => { - server.addPendingWorkflow('a', 'rtm'); - const { status } = await post('attempts/complete/a', { + server.enqueueAttempt(attempt1); + const { status } = await post('attempts/complete/attempt-1', { rtm_id: 'rtm', state: { x: 10, }, }); t.is(status, 200); - const result = server.getResult('a'); + const result = server.getResult('attempt-1'); t.deepEqual(result, { x: 10 }); }); test.serial('POST /attempts/complete - reject if unknown rtm', async (t) => { - const { status } = await post('attempts/complete/a', { + const { status } = await post('attempts/complete/attempt-1', { rtm_id: 'rtm', state: { x: 10, }, }); t.is(status, 400); - t.falsy(server.getResult('a')); + t.falsy(server.getResult('attempt-1')); }); test.serial( 'POST /attempts/complete - reject if unknown workflow', async (t) => { - server.addPendingWorkflow('b', 'rtm'); + server.enqueueAttempt({ id: 'b' }, 'rtm'); - const { status } = await post('attempts/complete/a', { + const { status } = await post('attempts/complete/attempt-1', { rtm_id: 'rtm', state: { x: 10, @@ -221,21 +220,21 @@ test.serial( }); t.is(status, 400); - t.falsy(server.getResult('a')); + t.falsy(server.getResult('attempt-1')); } ); test.serial('POST /attempts/complete - echo to event emitter', async (t) => { - server.addPendingWorkflow('a', 'rtm'); + server.enqueueAttempt(attempt1); let evt; let didCall = false; - server.once('workflow-complete', (e) => { + server.once('attempt-complete', (e) => { didCall = true; evt = e; }); - const { status } = await post('attempts/complete/a', { + const { status } = await post('attempts/complete/attempt-1', { rtm_id: 'rtm', state: { data: { @@ -248,7 +247,7 @@ test.serial('POST /attempts/complete - echo to event emitter', async (t) => { t.truthy(evt); t.is(evt.rtm_id, 'rtm'); - t.is(evt.workflow_id, 'a'); + t.is(evt.workflow_id, 'attempt-1'); t.deepEqual(evt.state, { data: { answer: 42 } }); }); From 9ee910a346a904c2156b9f51277123e4bb277555 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 8 Jun 2023 16:28:30 +0100 Subject: [PATCH 036/232] rtm: update tests --- .../rtm-server/src/mock/lightning/server.ts | 9 ++++----- .../rtm-server/src/mock/lightning/start.ts | 6 +++++- packages/rtm-server/test/integration.test.ts | 20 +------------------ .../rtm-server/test/mock/lightning.test.ts | 3 +-- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index c3c21e725..f3be1a8c2 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -3,7 +3,7 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import Router from '@koa/router'; -import createLogger, { LogLevel, Logger } from '@openfn/logger'; +import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; import createAPI from './api'; import createDevAPI from './api-dev'; @@ -22,14 +22,13 @@ export type ServerState = { export type LightningOptions = { logger?: Logger; logLevel?: LogLevel; - port?: string; + port?: string | number; }; // a mock lightning server const createLightningServer = (options: LightningOptions = {}) => { - const logger = - options.logger || - createLogger('LNG', { level: options.logLevel || 'info' }); + const logger = options.logger || createMockLogger(); + // App state const state = { credentials: {}, diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts index a53ba3d5d..7a81f3777 100644 --- a/packages/rtm-server/src/mock/lightning/start.ts +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -1,9 +1,13 @@ import createLightningServer from './server'; +import createLogger from '@openfn/logger'; -const port = 8888; +const port = '8888'; + +const logger = createLogger('LNG', { level: 'info' }); createLightningServer({ port, + logger, }); console.log('Started mock Lightning server on ', port); diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 6a23f4ddf..0962c03b6 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -27,7 +27,7 @@ test.before(() => { // Really high level test test.serial('process an attempt', async (t) => { - lng.addToQueue({ + lng.enqueueAttempt({ id: 'a1', jobs: [ { @@ -40,21 +40,3 @@ test.serial('process an attempt', async (t) => { const { state } = await lng.waitForResult('a1'); t.is(state.answer, 42); }); - -// process multiple attempts - -test.serial.skip( - 'should post to attempts/complete with the final state', - async (t) => { - // The mock RTM will evaluate the expression as JSON and return it - lng.addToQueue({ id: 'y', plan: [{ expression: '{ "answer": 42 }' }] }); - - await waitForEvent(rtm, 'workflow-complete'); - - // The RMT server will post to attempts/complete/:id with the state, which should eventually - // be available to our little debug API here - const result = await wait(() => lng.getResult('y')); - t.truthy(result); - t.is(result.answer, 42); - } -); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index d0fcf497b..517fab36d 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -8,8 +8,7 @@ const baseUrl = `http://localhost:8888${API_PREFIX}`; let server; test.before(() => { - const logger = createMockLogger(); - server = createLightningServer({ port: 8888, logger }); + server = createLightningServer({ port: 8888 }); }); test.afterEach(() => { From d5347080c7bcd1f9e8486cf2bf9794207bd9bf28 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 09:30:44 +0100 Subject: [PATCH 037/232] rtm-server: udpate readme --- packages/rtm-server/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/packages/rtm-server/README.md b/packages/rtm-server/README.md index f84e51141..f002b12b0 100644 --- a/packages/rtm-server/README.md +++ b/packages/rtm-server/README.md @@ -1,3 +1,33 @@ +# RTM Server + +The RTM server provides a HTTP interface between Lightning and a runtime manager (RTM). + +This package contains a mock Lightning implementation and a mock runtime manager implementation, allowing lightweight testing of the interfaces between them. + +The RTM server is designed for zero persistence. + +## Architecture + +Lightning will push an Attempt into a queue. + +The RTM server will greedily post Lightning to ask for outstanding attempts, which will returned as JSON objects to the server. + +The Lightning Attempt is converted ta Runtime Execution Plan, and passed to the RTM to execute. + +The server will listen to start, end, and log events in the RTM and POST them back to Lightning. + +## Dev server + +You can start a dev server by running + +``` +pnpm start:watch +``` + +This will wrap a real runtime manager instance into the server. It will rebuild when the server or RTM code changes. + +By default this does not connect to a lightning instance. [TODO need to enable this to talk to the default lightning mock server if it's enabled] + ## Lightning Mock You can start a Lightning mock server with: From 5ad22336355a2d889f33c6cb564e64acb5cc1abf Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 11:37:41 +0100 Subject: [PATCH 038/232] rtm-server: lightning api restructure --- packages/rtm-server/README.md | 4 ++++ .../rtm-server/src/mock/lightning/api-dev.ts | 6 +++--- packages/rtm-server/src/mock/lightning/api.ts | 16 +++++++++------- packages/rtm-server/src/mock/lightning/server.ts | 7 ++----- packages/rtm-server/src/mock/lightning/start.ts | 2 +- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/packages/rtm-server/README.md b/packages/rtm-server/README.md index f002b12b0..554cb9991 100644 --- a/packages/rtm-server/README.md +++ b/packages/rtm-server/README.md @@ -30,6 +30,10 @@ By default this does not connect to a lightning instance. [TODO need to enable t ## Lightning Mock +See `src/mock/lightning/api.ts` for an overview of the expected formal lightning API. This is the API that the RTM server will call. + +Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. + You can start a Lightning mock server with: ``` pnpm start:lightning diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index 5b140da9a..692c13cf3 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -80,7 +80,7 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger) => { // Set up some rest endpoints // Note that these are NOT prefixed -const setupRestAPI = (app: DevApp, state: ServerState, logger) => { +const setupRestAPI = (app: DevApp, _state: ServerState, logger: Logger) => { const router = new Router(); router.post('/attempt', (ctx) => { @@ -95,10 +95,10 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger) => { ctx.response.status = 200; }); - app.use(router.routes()); + return router.routes(); }; export default (app: DevApp, state: ServerState, logger: Logger) => { setupDevAPI(app, state, logger); - setupRestAPI(app, state, logger); + return setupRestAPI(app, state, logger); }; diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 3b19b37ab..b5056540a 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -1,4 +1,4 @@ -import type Router from '@koa/router'; +import Router from '@koa/router'; import { unimplemented, createListNextJob, @@ -9,6 +9,8 @@ import { } from './middleware'; import type { ServerState } from './server'; +import { API_PREFIX } from './server'; + interface RTMBody { rtm_id: string; } @@ -19,11 +21,11 @@ export interface AttemptCompleteBody extends RTMBody { state: any; // JSON state object (undefined? null?) } -export default (router: Router, logger, state: ServerState) => { - router.use(logger); - - // Basically all requests must include an rtm_id - // And probably later a security token +// Note that this API is hosted at api/1 +export default (state: ServerState) => { + const router = new Router({ prefix: API_PREFIX }); + // Basically all requests must include an rtm_id (And probably later a security token) + // TODO actually, is this an RTM Id or an RTM Server id? // POST attempts/next // Removes Attempts from the queue and returns them to the caller @@ -64,5 +66,5 @@ export default (router: Router, logger, state: ServerState) => { router.get('/workflows', unimplemented); router.get('/workflows/:id', unimplemented); - return router; + return router.routes(); }; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index f3be1a8c2..2b2036d5b 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -2,7 +2,6 @@ import { EventEmitter } from 'node:events'; import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; -import Router from '@koa/router'; import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; import createAPI from './api'; @@ -49,10 +48,8 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(klogger); // Mock API endpoints - const api = createAPI(new Router({ prefix: API_PREFIX }), klogger, state); - app.use(api.routes()); - - createDevAPI(app, state, logger); + app.use(createAPI(state)); + app.use(createDevAPI(app as any, state, logger)); const server = app.listen(options.port || 8888); app.destroy = () => { diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts index 7a81f3777..77218f830 100644 --- a/packages/rtm-server/src/mock/lightning/start.ts +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -10,4 +10,4 @@ createLightningServer({ logger, }); -console.log('Started mock Lightning server on ', port); +logger.success('Started mock Lightning server on ', port); From 02afdd0bddf1c00df1e4c5389c7b69f722ae9c2f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 11:38:37 +0100 Subject: [PATCH 039/232] rtm-server: log http stuff at debug --- packages/rtm-server/src/mock/lightning/server.ts | 2 +- packages/rtm-server/src/mock/lightning/start.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 2b2036d5b..958d290f4 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -44,7 +44,7 @@ const createLightningServer = (options: LightningOptions = {}) => { const app = new Koa(); app.use(bodyParser()); - const klogger = koaLogger((str) => logger.log(str)); + const klogger = koaLogger((str) => logger.debug(str)); app.use(klogger); // Mock API endpoints diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts index 77218f830..030d060f0 100644 --- a/packages/rtm-server/src/mock/lightning/start.ts +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -3,7 +3,7 @@ import createLogger from '@openfn/logger'; const port = '8888'; -const logger = createLogger('LNG', { level: 'info' }); +const logger = createLogger('LNG', { level: 'debug' }); createLightningServer({ port, From 22152e8d121d2dc6743133d56dfce6265baf9bbc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 12:15:02 +0100 Subject: [PATCH 040/232] rtm-server: sundry improvements, fix backoff --- packages/rtm-server/README.md | 8 ++++--- .../rtm-server/src/mock/lightning/start.ts | 24 +++++++++++++++---- packages/rtm-server/src/server.ts | 2 +- packages/rtm-server/src/start.ts | 16 ++++++++++--- .../rtm-server/src/util/try-with-backoff.ts | 24 ++++++++----------- packages/rtm-server/src/work-loop.ts | 11 +++++---- .../test/util/try-with-backoff.test.ts | 2 ++ 7 files changed, 57 insertions(+), 30 deletions(-) diff --git a/packages/rtm-server/README.md b/packages/rtm-server/README.md index 554cb9991..d39649a71 100644 --- a/packages/rtm-server/README.md +++ b/packages/rtm-server/README.md @@ -18,15 +18,17 @@ The server will listen to start, end, and log events in the RTM and POST them ba ## Dev server -You can start a dev server by running +You can start a dev server by running: ``` pnpm start:watch ``` -This will wrap a real runtime manager instance into the server. It will rebuild when the server or RTM code changes. +This will wrap a real runtime manager instance into the server. It will rebuild when the server or RTM code changes (although you'll have to `pnpm build:watch` in `runtime-manager`) -By default this does not connect to a lightning instance. [TODO need to enable this to talk to the default lightning mock server if it's enabled] +To connect to a lightning instance, pass the `-l` flag. Use `-l mock` to connect to the default mock server from this repo, or pass your own url. + +The server will create a Runtime Manager instance using the repo at `OPENFN_RTM_REPO_DIR` or `/tmp/openfn/repo`. ## Lightning Mock diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/rtm-server/src/mock/lightning/start.ts index 030d060f0..6cb424053 100644 --- a/packages/rtm-server/src/mock/lightning/start.ts +++ b/packages/rtm-server/src/mock/lightning/start.ts @@ -1,13 +1,29 @@ -import createLightningServer from './server'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; import createLogger from '@openfn/logger'; -const port = '8888'; +import createLightningServer from './server'; + +type Args = { + _: string[]; + port?: number; +}; + +const args = yargs(hideBin(process.argv)) + .command('server', 'Start a runtime manager server') + .option('port', { + alias: 'p', + description: 'Port to run the server on', + type: 'number', + default: 8888, + }) + .parse() as Args; const logger = createLogger('LNG', { level: 'debug' }); createLightningServer({ - port, + port: args.port, logger, }); -logger.success('Started mock Lightning server on ', port); +logger.success('Started mock Lightning server on ', args.port); diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index c4ad975b4..f0483c97a 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -95,7 +95,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { app.use(apiRouter.allowedMethods()); app.listen(port); - logger.info('Listening on', port); + logger.success('Listening on', port); (app as any).destroy = () => { // TODO close the work loop diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index 72e5c9b28..10a42e188 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -10,9 +10,10 @@ type Args = { _: string[]; port?: number; lightning?: string; + repoDir?: string; }; -const logger = createLogger('SRV'); +const logger = createLogger('SRV', { level: 'info' }); const args = yargs(hideBin(process.argv)) .command('server', 'Start a runtime manager server') @@ -24,11 +25,20 @@ const args = yargs(hideBin(process.argv)) }) .option('lightning', { alias: 'l', - description: 'Base url to Lightning', + description: + 'Base url to Lightning, eg, http://localhost:1234. Set to "mock" to use the default mock server', + }) + .option('repo-dir', { + alias: 'd', + description: 'Path to the runtime repo (where modules will be installed)', }) .parse() as Args; -const rtm = createRTM(); +if (args.lightning === 'mock') { + args.lightning = 'http://localhost:8888'; +} + +const rtm = createRTM('rtm', { repoDir: args.repoDir }); logger.debug('RTM created'); createRTMServer(rtm, { diff --git a/packages/rtm-server/src/util/try-with-backoff.ts b/packages/rtm-server/src/util/try-with-backoff.ts index a3c923fab..b3da9d864 100644 --- a/packages/rtm-server/src/util/try-with-backoff.ts +++ b/packages/rtm-server/src/util/try-with-backoff.ts @@ -1,6 +1,3 @@ -// re-usable function which will try a thing repeatedly -// TODO take a timeout - type Options = { attempts?: number; maxAttempts?: number; @@ -8,23 +5,21 @@ type Options = { timeout?: number; }; -// what is the API to this? -// Function should throw if it fails -// but in the main work loop it's not reall a fail for no work -// And we should back off -// under what circumstance should this function throw? -// If it timesout -// Can the inner function force a throw? An exit early? +const MAX_BACKOFF = 1000 * 60; + +// This function will try and call its first argument every {opts.timeout|100}ms +// If the function throws, it will "backoff" and try again a little later +// Right now it's a bit of a sketch, but it sort of works! const tryWithBackoff = (fn: any, opts: Options = {}) => { if (!opts.timeout) { - opts.timeout = 100; // TODO errors occur if this is too low? + opts.timeout = 100; } if (!opts.attempts) { opts.attempts = 1; } let { timeout, attempts, maxAttempts } = opts; - timeout = timeout || 1; - attempts = attempts || 1; + timeout = timeout; + attempts = attempts; return new Promise(async (resolve, reject) => { try { @@ -41,8 +36,9 @@ const tryWithBackoff = (fn: any, opts: Options = {}) => { const nextOpts = { maxAttempts, attempts: attempts + 1, - timeout: timeout * 2, + timeout: Math.min(MAX_BACKOFF, timeout * 1.2), }; + tryWithBackoff(fn, nextOpts).then(resolve).catch(reject); }, timeout); } diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index 7953a3c1a..9606e42d7 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -19,13 +19,14 @@ export default ( }, }); if (result.body) { - result.json().then((workflows) => { + const workflows = await result.json(); + if (workflows.length) { workflows.forEach(execute); - }); - return true; + return true; + } } - // return false to backoff and try again - return false; + // throw to backoff and try again + throw new Error('backoff'); }; const workLoop = () => { diff --git a/packages/rtm-server/test/util/try-with-backoff.test.ts b/packages/rtm-server/test/util/try-with-backoff.test.ts index b0c864567..b96d210c7 100644 --- a/packages/rtm-server/test/util/try-with-backoff.test.ts +++ b/packages/rtm-server/test/util/try-with-backoff.test.ts @@ -2,6 +2,8 @@ import test from 'ava'; import tryWithBackoff from '../../src/util/try-with-backoff'; +// TODO these unit tests are terrible and don't actually exercise the backoff or timeout interval + test('return immediately', async (t) => { let callCount = 0; const fn = async () => { From f5e4400681e71754047eaa8bebf909f207d875f4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 12:15:21 +0100 Subject: [PATCH 041/232] rtm: load repo from env var --- packages/runtime-manager/src/rtm.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 851e42535..f20345da0 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -70,9 +70,17 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const events = new EventEmitter(); if (!repoDir) { - repoDir = '/tmp/openfn/repo'; - logger.info('Defaulting repoDir to ', repoDir); + if (process.env.OPENFN_RTM_REPO_DIR) { + repoDir = process.env.OPENFN_RTM_REPO_DIR; + } else { + repoDir = '/tmp/openfn/repo'; + logger.warn('Using default repodir'); + logger.warn( + 'Set env var OPENFN_RTM_REPO_DIR to use a different directory' + ); + } } + logger.info('repoDir set to ', repoDir); const onWorkflowStarted = (workflowId: string, threadId: number) => { logger.info('starting workflow ', workflowId); From d04c1e66a72ab71757feaf1a1f1656f9036ca5e8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 14:35:33 +0100 Subject: [PATCH 042/232] runtime-manager: get autoinstall working --- packages/runtime-manager/src/rtm.ts | 23 +--- .../src/runners/autoinstall.ts | 82 ++++++++++++- .../test/runners/autoinstall.test.ts | 115 ++++++++++++++++++ 3 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 packages/runtime-manager/test/runners/autoinstall.test.ts diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index f20345da0..5a4a95757 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -10,6 +10,7 @@ import * as e from './events'; import createCompile from './runners/compile'; import createExecute from './runners/execute'; import createLogger, { JSONLog, Logger } from '@openfn/logger'; +import createAutoInstall from './runners/autoinstall'; export type State = any; // TODO I want a nice state def with generics @@ -146,6 +147,8 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { }); const compile = createCompile(logger, repoDir); + const autoinstall = createAutoInstall({ repoDir, logger }); + // How much of this happens inside the worker? // Shoud the main thread handle compilation? Has to if we want to cache // Unless we create a dedicated compiler worker @@ -159,7 +162,7 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { plan, }); - // TODO autoinstall + await autoinstall(plan); // Don't compile if we're running a mock (not a fan of this) const compiledPlan = noCompile ? plan : await compile(plan); @@ -176,29 +179,11 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { return result; }; - // const getActiveJobs = (): WorkflowStats[] => { - // const jobs = allWorkflows.map((id) => workflowList.get(id)); - // return jobs.filter((j) => j) as WorkflowStats[]; // no-op for typings - // }; - - // const getCompletedJobs = (): WorkflowStats[] => { - // return Array.from(allWorkflows.values()).filter((workflow) => workflow.status === 'done'); - // }; - - // const getErroredJobs = (): WorkflowStats[] => { - // return Array.from(workflowsList.values()).filter((workflow) => workflow.status === 'err'); - // }; - return { id, on: (type: string, fn: (...args: any[]) => void) => events.on(type, fn), once: (type: string, fn: (...args: any[]) => void) => events.once(type, fn), execute: handleExecute, - // getStatus, // no tests on this yet, not sure if I want to commit to it - - // getActiveJobs, - // getCompletedJobs, - // getErroredJobs, }; }; diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index 00dc73d1b..18c0aba48 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -1,9 +1,81 @@ // https://github.com/OpenFn/kit/issues/251 -const createAutoInstall = (options) => { - return async (plan) => { - // find all adaptors - // auto install what we need - // return when ready + +import { + ExecutionPlan, + ensureRepo, + getAliasedName, + loadRepoPkg, +} from '@openfn/runtime'; +import { install } from '@openfn/runtime'; +import type { Logger } from '@openfn/logger'; + +// The actual install function is not unit tested +// It's basically just a proxy to @openfn/runtime +const doHandleInstall = (specifier: string, options: Options) => + install(specifier, options.repoDir, options.logger); + +// The actual isInstalled function is not unit tested +// TODO this should probably all be handled (and tested) in @openfn/runtime +const doIsInstalled = async (specifier: string, options: Options) => { + const alias = getAliasedName(specifier); + // TODO is it really appropriate to load this file each time? + const pkg = await loadRepoPkg(options.repoDir); + if (pkg) { + const { dependencies } = pkg; + return dependencies.hasOwnProperty(alias); + } +}; + +export const identifyAdaptors = (plan: ExecutionPlan): Set => { + const adaptors = new Set(); + plan.jobs + .filter((job) => job.adaptor) + .forEach((job) => adaptors.add(job.adaptor!)); + return adaptors; +}; + +type Options = { + repoDir: string; + logger: Logger; + handleInstall?( + fn: string, + options?: Pick + ): Promise; + handleIsInstalled?( + fn: string, + options?: Pick + ): Promise; +}; + +const createAutoInstall = (options: Options) => { + const install = options.handleInstall || doHandleInstall; + const isInstalled = options.handleIsInstalled || doIsInstalled; + const pending: Record> = {}; + + let didValidateRepo = false; + + return async (plan: ExecutionPlan): Promise => { + if (!didValidateRepo && options.repoDir) { + // TODO what if this throws? + // Whole server probably needs to crash, so throwing is probably appropriate + await ensureRepo(options.repoDir, options.logger); + didValidateRepo = true; + } + + const adaptors = identifyAdaptors(plan); + // TODO would rather do all this in parallel but this is fine for now + // TODO set iteration is weirdly difficult? + for (const a of Array.from(adaptors)) { + const needsInstalling = !(await isInstalled(a, options)); + if (needsInstalling) { + if (!pending[a]) { + // add a promise to the pending array + pending[a] = install(a, options); + } + // Return the pending promise (safe to do this multiple times) + await pending[a].then(); + } + } }; }; diff --git a/packages/runtime-manager/test/runners/autoinstall.test.ts b/packages/runtime-manager/test/runners/autoinstall.test.ts new file mode 100644 index 000000000..bfe45d4b1 --- /dev/null +++ b/packages/runtime-manager/test/runners/autoinstall.test.ts @@ -0,0 +1,115 @@ +import test from 'ava'; +import createAutoInstall, { + identifyAdaptors, +} from '../../src/runners/autoinstall'; + +const mockIsInstalled = (pkg) => async (specifier: string) => { + const alias = specifier.split('@').join('_'); + return pkg.dependencies.hasOwnProperty(alias); +}; + +// TODO should this write to package json? +// I don't think there's any need +const mockHandleInstall = async (specifier: string): Promise => + new Promise((r) => r()).then(); + +test('mock is installed: should be installed', async (t) => { + const isInstalled = mockIsInstalled({ + name: 'repo', + dependencies: { + 'x_1.0.0': 'path', + }, + }); + + const result = await isInstalled('x@1.0.0'); + t.true(result); +}); + +test('mock is installed: should not be installed', async (t) => { + const isInstalled = mockIsInstalled({ + name: 'repo', + dependencies: { + 'x_1.0.0': 'path', + }, + }); + + const result = await isInstalled('x@1.0.1'); + t.false(result); +}); + +test('mock install: should return async', async (t) => { + await mockHandleInstall('x@1.0.0'); + t.true(true); +}); + +test('identifyAdaptors: pick out adaptors and remove duplicates', (t) => { + const plan = { + jobs: [ + { + adaptor: 'common@1.0.0', + }, + { + adaptor: 'common@1.0.0', + }, + { + adaptor: 'common@1.0.1', + }, + ], + }; + const adaptors = identifyAdaptors(plan); + t.true(adaptors.size === 2); + t.true(adaptors.has('common@1.0.0')); + t.true(adaptors.has('common@1.0.1')); +}); + +// This doesn't do anything except check that the mocks are installed +test('autoinstall: should call both mock functions', (t) => { + let didCallIsInstalled = false; + let didCallInstall = true; + + const mockIsInstalled = async () => { + didCallIsInstalled = true; + return false; + }; + const mockInstall = async () => { + didCallInstall = true; + return; + }; + + const autoinstall = createAutoInstall({ + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }); + + autoinstall({ + jobs: [{ adaptor: 'x@1.0.0' }], + }); + + t.true(didCallIsInstalled); + t.true(didCallInstall); +}); + +test('autoinstall: only call install once if there are two concurrent install requests', async (t) => { + let callCount = 0; + + const mockInstall = (specififer: string) => + new Promise((resolve) => { + callCount++; + setTimeout(() => resolve(), 20); + }); + + const autoinstall = createAutoInstall({ + handleInstall: mockInstall, + handleIsInstalled: async () => false, + }); + + autoinstall({ + jobs: [{ adaptor: 'x@1.0.0' }], + }); + + await autoinstall({ + jobs: [{ adaptor: 'x@1.0.0' }], + }); + + t.is(callCount, 1); +}); From bb2bc1f6a319e7cee2c492ae6da313ab4fcaf49e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 14:52:31 +0100 Subject: [PATCH 043/232] rtm: fix autoinstall, prefix local logs with workflowid --- packages/runtime-manager/src/rtm.ts | 8 +++++++- packages/runtime-manager/src/runners/autoinstall.ts | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 5a4a95757..98bd16984 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -133,7 +133,13 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { // Seamlessly proxy the log to the local stdout // TODO runtime logging probably needs to be at info level? // Debug information is mostly irrelevant for lightning - logger.proxy(message); + const newMessage = { + ...message, + // Prefix the job id in all local jobs + // I'm sure there are nicer, more elegant ways of doing this + message: [`[${workflowId}]`, ...message.message], + }; + logger.proxy(newMessage); events.emit(e.WORKFLOW_LOG, { workflowId, message, diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index 18c0aba48..bf20f627f 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -18,10 +18,22 @@ const doHandleInstall = (specifier: string, options: Options) => // TODO this should probably all be handled (and tested) in @openfn/runtime const doIsInstalled = async (specifier: string, options: Options) => { const alias = getAliasedName(specifier); + if (!alias.match('_')) { + // Note that if the adaptor has no version number, the alias will be "wrong" + // and we will count the adaptor as uninstalled + // The install function will later decide a version number and may, or may + // not, install for us. + // This log isn't terrible helpful as there's no attempt version info + options.logger.warn( + `adaptor ${specifier} does not have a version number - will attempt to auto-install` + ); + } // TODO is it really appropriate to load this file each time? const pkg = await loadRepoPkg(options.repoDir); if (pkg) { const { dependencies } = pkg; + console.log(alias); + console.log(dependencies); return dependencies.hasOwnProperty(alias); } }; From e9819dc2591ee1b56e4fe000e527cb61bf3d9702 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 15:00:59 +0100 Subject: [PATCH 044/232] rtm: logging fixes --- packages/rtm-server/src/server.ts | 14 ++++++++------ packages/runtime-manager/src/rtm.ts | 1 - 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index f0483c97a..a904bb632 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -109,12 +109,14 @@ function createServer(rtm: any, options: ServerOptions = {}) { logger.warn('No lightning URL provided'); } - // TODO how about an 'all' so we can "route" events? - rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { - logger.log(`${id}: workflow complete: `, id); - logger.log(state); - postResult(rtm.id, options.lightning!, id, state); - }); + rtm.on( + 'workflow-complete', + ({ workflowId, state }: { workflowId: string; state: any }) => { + logger.log(`workflow complete: `, workflowId); + logger.log(state); + postResult(rtm.id, options.lightning!, workflowId, state); + } + ); rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { logger.log(`${id}: `, ...messages); diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index 98bd16984..d32a7af01 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -175,7 +175,6 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { logger.debug('workflow compiled ', plan.id); const result = await execute(compiledPlan); - logger.success(result); completeWorkflow(plan.id!, result); logger.debug('finished executing workflow ', plan.id); From f759648177f26aa0c44f8d3f0e634639ef0ce47f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 15:02:04 +0100 Subject: [PATCH 045/232] rtm: remoe debug log --- packages/runtime-manager/src/runners/autoinstall.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index bf20f627f..0d65b7f2c 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -32,8 +32,6 @@ const doIsInstalled = async (specifier: string, options: Options) => { const pkg = await loadRepoPkg(options.repoDir); if (pkg) { const { dependencies } = pkg; - console.log(alias); - console.log(dependencies); return dependencies.hasOwnProperty(alias); } }; From 2943ca81ed2d62ddec982815adaf7b6a331c8383 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 15:12:16 +0100 Subject: [PATCH 046/232] rtm-server: convert initial state on attempt to data --- packages/rtm-server/src/types.d.ts | 1 + packages/rtm-server/src/util/convert-attempt.ts | 5 +++++ .../test/util/convert-attempt.test.ts | 17 +++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index 5831d2347..476d7f1cb 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -18,6 +18,7 @@ export type Node = { adaptor?: string; credential?: any; // TODO tighten this up, string or object type?: 'webhook' | 'cron'; // trigger only + state?: any; // Initial state / defaults }; export interface Edge { diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/rtm-server/src/util/convert-attempt.ts index 1ca9f7664..befc6c7d3 100644 --- a/packages/rtm-server/src/util/convert-attempt.ts +++ b/packages/rtm-server/src/util/convert-attempt.ts @@ -44,6 +44,11 @@ export default (attempt: Attempt): ExecutionPlan => { adaptor: job.adaptor, }; + if (job.state) { + // TODO this is likely to change + nodes[id].data = job.state; + } + const next = edges .filter((e) => e.source_job_id === id) .reduce((obj, edge) => { diff --git a/packages/rtm-server/test/util/convert-attempt.test.ts b/packages/rtm-server/test/util/convert-attempt.test.ts index adb94e506..d2e6cc6e3 100644 --- a/packages/rtm-server/test/util/convert-attempt.test.ts +++ b/packages/rtm-server/test/util/convert-attempt.test.ts @@ -51,6 +51,23 @@ test('convert a single job', (t) => { }); }); +// Note idk how lightningg will handle state/defaults on a job +// but this is what we'll do right now +test('convert a single job with data', (t) => { + const attempt: Attempt = { + id: 'w', + jobs: [createNode({ state: { data: { x: 22 } } })], + triggers: [], + edges: [], + }; + const result = convertAttempt(attempt); + + t.deepEqual(result, { + id: 'w', + jobs: [createJob({ data: { data: { x: 22 } } })], + }); +}); + test('Accept a partial attempt object', (t) => { const attempt: Partial = { id: 'w', From 706ed051032f43b67ea107444c9f8d058cef4576 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:00:04 +0100 Subject: [PATCH 047/232] rtm: properly map adaptor versions for the linker --- packages/runtime-manager/src/rtm.ts | 6 ++--- .../src/runners/autoinstall.ts | 17 +++++++++--- .../runtime-manager/src/runners/execute.ts | 13 ++++------ packages/runtime-manager/src/worker.ts | 8 +++--- .../test/runners/autoinstall.test.ts | 26 +++++++++++++++++++ 5 files changed, 53 insertions(+), 17 deletions(-) diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index d32a7af01..c639a6cc4 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -147,7 +147,7 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { }; // Create "runner" functions for execute and compile - const execute = createExecute(workers, repoDir, logger, { + const execute = createExecute(workers, logger, { start: onWorkflowStarted, log: onWorkflowLog, }); @@ -168,13 +168,13 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { plan, }); - await autoinstall(plan); + const adaptorPaths = await autoinstall(plan); // Don't compile if we're running a mock (not a fan of this) const compiledPlan = noCompile ? plan : await compile(plan); logger.debug('workflow compiled ', plan.id); - const result = await execute(compiledPlan); + const result = await execute(compiledPlan, adaptorPaths); completeWorkflow(plan.id!, result); logger.debug('finished executing workflow ', plan.id); diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index 0d65b7f2c..cd75cf355 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -4,6 +4,7 @@ import { ExecutionPlan, ensureRepo, getAliasedName, + getNameAndVersion, loadRepoPkg, } from '@openfn/runtime'; import { install } from '@openfn/runtime'; @@ -57,6 +58,8 @@ type Options = { ): Promise; }; +export type ModulePaths = Record; + const createAutoInstall = (options: Options) => { const install = options.handleInstall || doHandleInstall; const isInstalled = options.handleIsInstalled || doIsInstalled; @@ -64,7 +67,7 @@ const createAutoInstall = (options: Options) => { let didValidateRepo = false; - return async (plan: ExecutionPlan): Promise => { + return async (plan: ExecutionPlan): Promise => { if (!didValidateRepo && options.repoDir) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate @@ -72,10 +75,17 @@ const createAutoInstall = (options: Options) => { didValidateRepo = true; } - const adaptors = identifyAdaptors(plan); + const adaptors = Array.from(identifyAdaptors(plan)); // TODO would rather do all this in parallel but this is fine for now // TODO set iteration is weirdly difficult? - for (const a of Array.from(adaptors)) { + const paths: ModulePaths = {}; + for (const a of adaptors) { + // Return a path name to this module for the linker to use later + // TODO this is all a bit rushed + const alias = getAliasedName(a); + const { name } = getNameAndVersion(a); + paths[name] = { path: `${options.repoDir}/node_modules/${alias}` }; + const needsInstalling = !(await isInstalled(a, options)); if (needsInstalling) { if (!pending[a]) { @@ -86,6 +96,7 @@ const createAutoInstall = (options: Options) => { await pending[a].then(); } } + return paths; }; }; diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/runtime-manager/src/runners/execute.ts index d7154d74b..69c1fb24d 100644 --- a/packages/runtime-manager/src/runners/execute.ts +++ b/packages/runtime-manager/src/runners/execute.ts @@ -2,23 +2,20 @@ import type { WorkerPool } from 'workerpool'; import type { ExecutionPlan } from '@openfn/runtime'; import { Logger } from '@openfn/logger'; + import * as e from '../events'; +import { ModulePaths } from './autoinstall'; // A lot of callbacks needed here // Is it better to just return the handler? // But then this function really isn't doing so much // (I guess that's true anyway) -const execute = ( - workers: WorkerPool, - repoDir: string, - logger: Logger, - events: any -) => { +const execute = (workers: WorkerPool, logger: Logger, events: any) => { const { start, log, error } = events; - return (plan: ExecutionPlan) => { + return (plan: ExecutionPlan, adaptorPaths: ModulePaths) => { return new Promise((resolve) => workers - .exec('run', [plan, repoDir], { + .exec('run', [plan, adaptorPaths], { on: ({ type, ...args }: e.WorkflowEvent) => { if (type === e.WORKFLOW_START) { const { workflowId, threadId } = args as e.AcceptWorkflowEvent; diff --git a/packages/runtime-manager/src/worker.ts b/packages/runtime-manager/src/worker.ts index b91fa2ef3..b4f1944b4 100644 --- a/packages/runtime-manager/src/worker.ts +++ b/packages/runtime-manager/src/worker.ts @@ -18,14 +18,16 @@ import type { ExecutionPlan } from '@openfn/runtime'; import helper, { createLoggers } from './worker-helper'; workerpool.worker({ - run: (plan: ExecutionPlan, repoDir: string) => { + run: ( + plan: ExecutionPlan, + adaptorPaths: Record + ) => { const { logger, jobLogger } = createLoggers(plan.id!); - const options = { logger, jobLogger, linker: { - repo: repoDir, + modules: adaptorPaths, }, }; diff --git a/packages/runtime-manager/test/runners/autoinstall.test.ts b/packages/runtime-manager/test/runners/autoinstall.test.ts index bfe45d4b1..844bfc560 100644 --- a/packages/runtime-manager/test/runners/autoinstall.test.ts +++ b/packages/runtime-manager/test/runners/autoinstall.test.ts @@ -113,3 +113,29 @@ test('autoinstall: only call install once if there are two concurrent install re t.is(callCount, 1); }); + +test.only('autoinstall: return a map to modules', async (t) => { + const plan = { + // Note that we have difficulty now if a workflow imports two versions of the same adaptor + jobs: [ + { + adaptor: 'common@1.0.0', + }, + { + adaptor: 'http@1.0.0', + }, + ], + }; + + const autoinstall = createAutoInstall({ + repoDir: 'a/b/c', + handleInstall: async () => true, + handleIsInstalled: async () => false, + }); + + const result = await autoinstall(plan); + t.deepEqual(result, { + common: { path: 'a/b/c/node_modules/common_1.0.0' }, + http: { path: 'a/b/c/node_modules/http_1.0.0' }, + }); +}); From d3ca452da34d784ecc16c8e3a25b300b2b328482 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:12:19 +0100 Subject: [PATCH 048/232] rtm: fix complete event --- packages/rtm-server/src/server.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index a904bb632..6bc9fb6fd 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -109,14 +109,11 @@ function createServer(rtm: any, options: ServerOptions = {}) { logger.warn('No lightning URL provided'); } - rtm.on( - 'workflow-complete', - ({ workflowId, state }: { workflowId: string; state: any }) => { - logger.log(`workflow complete: `, workflowId); - logger.log(state); - postResult(rtm.id, options.lightning!, workflowId, state); - } - ); + rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { + logger.log(`workflow complete: `, id); + logger.log(state); + postResult(rtm.id, options.lightning!, id, state); + }); rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { logger.log(`${id}: `, ...messages); From c76a579ecc4e565d7729f35e6165257c0b70d62d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:25:21 +0100 Subject: [PATCH 049/232] rtm: skip repo validation in unit tests --- packages/runtime-manager/src/runners/autoinstall.ts | 4 +++- packages/runtime-manager/test/runners/autoinstall.test.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index cd75cf355..d52e7cf06 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -48,6 +48,7 @@ export const identifyAdaptors = (plan: ExecutionPlan): Set => { type Options = { repoDir: string; logger: Logger; + skipRepoValidation: boolean; handleInstall?( fn: string, options?: Pick @@ -66,9 +67,10 @@ const createAutoInstall = (options: Options) => { const pending: Record> = {}; let didValidateRepo = false; + const { skipRepoValidation } = options; return async (plan: ExecutionPlan): Promise => { - if (!didValidateRepo && options.repoDir) { + if (!skipRepoValidation && !didValidateRepo && options.repoDir) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate await ensureRepo(options.repoDir, options.logger); diff --git a/packages/runtime-manager/test/runners/autoinstall.test.ts b/packages/runtime-manager/test/runners/autoinstall.test.ts index 844bfc560..d32bc547b 100644 --- a/packages/runtime-manager/test/runners/autoinstall.test.ts +++ b/packages/runtime-manager/test/runners/autoinstall.test.ts @@ -114,7 +114,7 @@ test('autoinstall: only call install once if there are two concurrent install re t.is(callCount, 1); }); -test.only('autoinstall: return a map to modules', async (t) => { +test('autoinstall: return a map to modules', async (t) => { const plan = { // Note that we have difficulty now if a workflow imports two versions of the same adaptor jobs: [ @@ -129,6 +129,7 @@ test.only('autoinstall: return a map to modules', async (t) => { const autoinstall = createAutoInstall({ repoDir: 'a/b/c', + skipRepoValidation: true, handleInstall: async () => true, handleIsInstalled: async () => false, }); From 0441027f83e066111bbe15d5abc99fa176b1e5a3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:26:47 +0100 Subject: [PATCH 050/232] rtm-server: update attempt data structure --- packages/rtm-server/src/util/convert-attempt.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/rtm-server/src/util/convert-attempt.ts index befc6c7d3..382b0e2a2 100644 --- a/packages/rtm-server/src/util/convert-attempt.ts +++ b/packages/rtm-server/src/util/convert-attempt.ts @@ -46,7 +46,7 @@ export default (attempt: Attempt): ExecutionPlan => { if (job.state) { // TODO this is likely to change - nodes[id].data = job.state; + nodes[id].state = job.state; } const next = edges From 462e2674b4fc7c3d0584aeacde9104102f2f72a8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:35:35 +0100 Subject: [PATCH 051/232] rtm: another complete event fix --- packages/runtime-manager/src/rtm.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index c639a6cc4..b21d0bbcf 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -121,7 +121,7 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { // forward the event on to any external listeners events.emit(e.WORKFLOW_COMPLETE, { - workflowId, + id: workflowId, duration: workflow.duration, state, }); @@ -175,7 +175,7 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { logger.debug('workflow compiled ', plan.id); const result = await execute(compiledPlan, adaptorPaths); - completeWorkflow(plan.id!, result); + completeWorkflow(compiledPlan.id!, result); logger.debug('finished executing workflow ', plan.id); // Return the result From 3372e192a5d25bc05ea8103d59c043c900a880aa Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:38:45 +0100 Subject: [PATCH 052/232] tweak dependencies --- packages/runtime-manager/package.json | 4 +--- .../runtime-manager/src/runners/autoinstall.ts | 2 +- pnpm-lock.yaml | 16 ++++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/runtime-manager/package.json b/packages/runtime-manager/package.json index 50ace3b70..2721a3226 100644 --- a/packages/runtime-manager/package.json +++ b/packages/runtime-manager/package.json @@ -14,12 +14,10 @@ "author": "Open Function Group ", "license": "ISC", "dependencies": { - "@openfn/compiler": "workspace:^0.0.34", + "@openfn/compiler": "workspace:*", "@openfn/language-common": "2.0.0-rc3", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", - "@types/koa": "^2.13.5", - "@types/workerpool": "^6.1.0", "koa": "^2.13.4", "workerpool": "^6.2.1" }, diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/runtime-manager/src/runners/autoinstall.ts index d52e7cf06..96084e14d 100644 --- a/packages/runtime-manager/src/runners/autoinstall.ts +++ b/packages/runtime-manager/src/runners/autoinstall.ts @@ -48,7 +48,7 @@ export const identifyAdaptors = (plan: ExecutionPlan): Set => { type Options = { repoDir: string; logger: Logger; - skipRepoValidation: boolean; + skipRepoValidation?: boolean; handleInstall?( fn: string, options?: Pick diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d34c260e1..8c9fa2ee9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -428,7 +428,7 @@ importers: packages/runtime-manager: dependencies: '@openfn/compiler': - specifier: workspace:^0.0.34 + specifier: workspace:* version: link:../compiler '@openfn/language-common': specifier: 2.0.0-rc3 @@ -439,12 +439,6 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime - '@types/koa': - specifier: ^2.13.5 - version: 2.13.5 - '@types/workerpool': - specifier: ^6.1.0 - version: 6.1.0 koa: specifier: ^2.13.4 version: 2.13.4 @@ -452,9 +446,15 @@ importers: specifier: ^6.2.1 version: 6.2.1 devDependencies: + '@types/koa': + specifier: ^2.13.5 + version: 2.13.5 '@types/node': specifier: ^17.0.31 version: 17.0.45 + '@types/workerpool': + specifier: ^6.1.0 + version: 6.1.0 ava: specifier: 5.1.0 version: 5.1.0 @@ -1426,7 +1426,7 @@ packages: resolution: {integrity: sha512-C+J/c1BHyc351xJuiH2Jbe+V9hjf5mCzRP0UK4KEpF5SpuU+vJ/FC5GLZsCU/PJpp/3I6Uwtfm3DG7Lmrb7LOQ==} dependencies: '@types/node': 18.15.3 - dev: false + dev: true /@types/wrap-ansi@3.0.0: resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} From 5906a86e0748fb7ffa9ee8fcebee7584234ed014 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:41:44 +0100 Subject: [PATCH 053/232] rtm: update test --- packages/runtime-manager/test/rtm.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/runtime-manager/test/rtm.test.ts index 83929337a..61fbdb716 100644 --- a/packages/runtime-manager/test/rtm.test.ts +++ b/packages/runtime-manager/test/rtm.test.ts @@ -85,7 +85,7 @@ test('events: workflow-complete', async (t) => { await rtm.execute(plan); t.true(didCall); - t.is(evt.workflowId, plan.id); + t.is(evt.id, plan.id); t.assert(!isNaN(evt.duration)); t.deepEqual(evt.state, { data: { answer: 42 } }); }); From dda62a6be196fcf7cedda6f2214ff71272b35481 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:44:16 +0100 Subject: [PATCH 054/232] rtm-server: update test --- packages/rtm-server/test/util/convert-attempt.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rtm-server/test/util/convert-attempt.test.ts b/packages/rtm-server/test/util/convert-attempt.test.ts index d2e6cc6e3..24d6f9d65 100644 --- a/packages/rtm-server/test/util/convert-attempt.test.ts +++ b/packages/rtm-server/test/util/convert-attempt.test.ts @@ -64,7 +64,7 @@ test('convert a single job with data', (t) => { t.deepEqual(result, { id: 'w', - jobs: [createJob({ data: { data: { x: 22 } } })], + jobs: [createJob({ state: { data: { x: 22 } } })], }); }); From 485905b97dc69ee488987a525c0b37d81de95bcb Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jun 2023 16:51:17 +0100 Subject: [PATCH 055/232] changesets --- .changeset/selfish-nails-live.md | 5 +++++ .changeset/shaggy-jars-brake.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/selfish-nails-live.md create mode 100644 .changeset/shaggy-jars-brake.md diff --git a/.changeset/selfish-nails-live.md b/.changeset/selfish-nails-live.md new file mode 100644 index 000000000..a0b19b094 --- /dev/null +++ b/.changeset/selfish-nails-live.md @@ -0,0 +1,5 @@ +--- +'@openfn/rtm-server': minor +--- + +First pass of runtime manager server" diff --git a/.changeset/shaggy-jars-brake.md b/.changeset/shaggy-jars-brake.md new file mode 100644 index 000000000..f6f4abe41 --- /dev/null +++ b/.changeset/shaggy-jars-brake.md @@ -0,0 +1,5 @@ +--- +'@openfn/runtime-manager': patch +--- + +Update runtime manager to handle workflows and mach new design From 987596e3bc8887de32acb2b43b9de395ba3ed5e1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 11 Jul 2023 13:04:05 +0100 Subject: [PATCH 056/232] rtm-server: update nodemon --- packages/rtm-server/package.json | 2 +- pnpm-lock.yaml | 28 ++++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index d7764bc9a..c5d4fc65d 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -33,7 +33,7 @@ "@types/node": "^18.15.3", "@types/yargs": "^17.0.12", "ava": "5.1.0", - "nodemon": "^2.0.19", + "nodemon": "3.0.1", "ts-node": "^10.9.1", "tslib": "^2.4.0", "tsup": "^6.2.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8c9fa2ee9..0f44d5dc4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -370,8 +370,8 @@ importers: specifier: 5.1.0 version: 5.1.0 nodemon: - specifier: ^2.0.19 - version: 2.0.19 + specifier: 3.0.1 + version: 3.0.1 ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@18.15.3)(typescript@4.8.3) @@ -4593,6 +4593,23 @@ packages: undefsafe: 2.0.5 dev: true + /nodemon@3.0.1: + resolution: {integrity: sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==} + engines: {node: '>=10'} + hasBin: true + dependencies: + chokidar: 3.5.3 + debug: 3.2.7(supports-color@5.5.0) + ignore-by-default: 1.0.1 + minimatch: 3.1.2 + pstree.remy: 1.1.8 + semver: 7.5.4 + simple-update-notifier: 2.0.0 + supports-color: 5.5.0 + touch: 3.1.0 + undefsafe: 2.0.5 + dev: true + /nofilter@3.1.0: resolution: {integrity: sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==} engines: {node: '>=12.19'} @@ -5536,6 +5553,13 @@ packages: semver: 7.0.0 dev: true + /simple-update-notifier@2.0.0: + resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} + engines: {node: '>=10'} + dependencies: + semver: 7.5.4 + dev: true + /slash@3.0.0: resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} engines: {node: '>=8'} From 13d7699e99fbad28fddfa8b3c745994b52893b79 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 27 Jul 2023 09:30:35 +0100 Subject: [PATCH 057/232] rtm: update readme --- packages/runtime-manager/README.md | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/runtime-manager/README.md b/packages/runtime-manager/README.md index 48cc52c5b..7335ff158 100644 --- a/packages/runtime-manager/README.md +++ b/packages/runtime-manager/README.md @@ -4,17 +4,6 @@ An example runtime manager service. The runtime manager is designed as a long running node service that runs jobs as worker threads. -## Demo Server - -Run `pnpm start` to start the manager as a web service. This gives a bit of an example of how the manager might be used. - -Go to `localhost:1234` to see a report on any active threads as well as the job history. - -Post anything to the server to run the test job. The test job will run for a random number of seconds and return a random number. Patent pending. - -The server will report usage statistics when any job finishes. -~~Post to `/job` to spin out a new job.~~ - ## Usage To integrate the manager into your own application: From 588cd0d42e1dcb29199f3ec2aecc86e32847b8eb Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 31 Aug 2023 14:30:21 +0100 Subject: [PATCH 058/232] rtm: make private --- packages/runtime-manager/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/runtime-manager/package.json b/packages/runtime-manager/package.json index 2008cfb89..da62d8d40 100644 --- a/packages/runtime-manager/package.json +++ b/packages/runtime-manager/package.json @@ -4,6 +4,7 @@ "description": "An example runtime manager service.", "main": "dist/index.js", "type": "module", + "private": true, "scripts": { "test": "pnpm ava", "test:types": "pnpm tsc --noEmit --project tsconfig.json", From 92528ba9eb534cb9104f8c8b0e2e4ff925bbf37a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 13 Sep 2023 17:33:33 +0100 Subject: [PATCH 059/232] rtm: add mock websockets --- packages/rtm-server/package.json | 4 +- packages/rtm-server/sockets.js | 52 ++ packages/rtm-server/sockets2.js | 8 + packages/rtm-server/src/mock/lightning/api.ts | 31 + packages/rtm-server/src/mock/socket-server.ts | 76 +++ packages/rtm-server/src/server.ts | 11 +- .../test/mock/socket-server.test.ts | 50 ++ packages/rtm-server/test/server.test.ts | 33 +- pnpm-lock.yaml | 545 +++++------------- 9 files changed, 417 insertions(+), 393 deletions(-) create mode 100644 packages/rtm-server/sockets.js create mode 100644 packages/rtm-server/sockets2.js create mode 100644 packages/rtm-server/src/mock/socket-server.ts create mode 100644 packages/rtm-server/test/mock/socket-server.test.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index c5d4fc65d..c9d20f73e 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -24,7 +24,9 @@ "@types/koa-logger": "^3.1.2", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", - "koa-logger": "^3.2.1" + "koa-logger": "^3.2.1", + "phoenix-channels": "^1.0.0", + "ws": "^8.14.1" }, "devDependencies": { "@types/koa": "^2.13.5", diff --git a/packages/rtm-server/sockets.js b/packages/rtm-server/sockets.js new file mode 100644 index 000000000..4d074996a --- /dev/null +++ b/packages/rtm-server/sockets.js @@ -0,0 +1,52 @@ +import WebSocket, { WebSocketServer } from 'ws'; +import phx from 'phoenix-channels'; +import http from 'node:http'; + +const { Socket } = phx; + +/* + * web socket experiments + */ + +/* + Super simple ws implementation +*/ +const wsServer = new WebSocketServer({ + port: 8080, +}); + +wsServer.on('connection', function (ws) { + console.log('connection'); + + ws.on('message', function (data) { + console.log('server received: %s', data); + + // TMP + // process.exit(0); + }); +}); + +// const s = new WebSocket('ws://localhost:8080'); + +// // This bit is super important! Can't send ontil we've got the on open callback +// s.on('open', () => { +// console.log('sending...'); +// s.send('hello'); +// }); + +// This is a phoenix socket backing onto a normal websocket server +const s = new Socket('ws://localhost:8080'); +s.connect(); + +console.log('*'); +let channel = s.channel('room:lobby', {}); +channel.join().receive('ok', (resp) => { + console.log('Joined successfully', resp); + + channel.push('hello'); +}); +// .receive('error', (resp) => { +// console.log('Unable to join', resp); +// }); + +// channel.push('hello'); diff --git a/packages/rtm-server/sockets2.js b/packages/rtm-server/sockets2.js new file mode 100644 index 000000000..536f171fa --- /dev/null +++ b/packages/rtm-server/sockets2.js @@ -0,0 +1,8 @@ +import WebSocket, { WebSocketServer } from 'ws'; + +const s = new WebSocket('ws://localhost:8080'); +s.on('open', () => { + console.log('sending...'); + s.send('hello'); + process.exit(0); +}); diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index b5056540a..9c80b4631 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -21,7 +21,38 @@ export interface AttemptCompleteBody extends RTMBody { state: any; // JSON state object (undefined? null?) } +// this new API is websocket based +// Events map to handlers +// can I even implement this in JS? Not with pheonix anyway. hmm. +// dead at the first hurdle really. +// what if I do the server side mock in koa, can I use the pheonix client to connect? +const newApi = () => { + const noop = () => {}; + + // This may actually get split into a server bit and a an attempts bit, reflecting the different channels + const events = { + hello: noop, + + 'attempt:claim': noop, + 'attempt:start': noop, + 'attempt:complete': noop, + 'attempt:get_credential': noop, + 'attempt:credential': noop, + 'attempt:get_dataclip': noop, + 'attempt:dataclip': noop, + + 'run:start': noop, + 'run:end': noop, + 'run:log': noop, + }; + + const handleEvent = (name) => {}; + + return handleEvent; +}; + // Note that this API is hosted at api/1 +// Deprecated export default (state: ServerState) => { const router = new Router({ prefix: API_PREFIX }); // Basically all requests must include an rtm_id (And probably later a security token) diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/socket-server.ts new file mode 100644 index 000000000..369895e33 --- /dev/null +++ b/packages/rtm-server/src/mock/socket-server.ts @@ -0,0 +1,76 @@ +import { WebSocketServer } from 'ws'; +// mock pheonix websocket server + +// - route messages to rooms +// - respond ok to connections +type Topic = string; + +type WS = any; + +type PhoenixEvent = { + topic: Topic; + event: string; + payload?: any; + ref?: string; +}; + +type EventHandler = (event: string, payload: any) => void; + +function createServer() { + const channels: Record = {}; + + const wsServer = new WebSocketServer({ + port: 8080, + }); + + // util to send a response to a particular topic + const reply = () => {}; + + const events = { + // When joining a channel, we need to send a chan_reply_{ref} message back to the socket + phx_join: (ws, { topic, ref }) => { + ws.send( + JSON.stringify({ + // here is the magic reply event + // see channel.replyEventName + event: `chan_reply_${ref}`, + topic, + payload: { status: 'ok', response: 'ok' }, + ref, + }) + ); + }, + }; + + wsServer.on('connection', function (ws: WS) { + ws.on('message', function (data: string) { + const evt = JSON.parse(data) as PhoenixEvent; + if (evt.topic) { + // phx sends this info in each message + const { topic, event, payload, ref } = evt; + + if (events[event]) { + // handle system/phoenix events + events[event](ws, { topic, payload, ref }); + } else { + // handle custom/user events + if (channels[topic]) { + channels[topic].forEach((fn) => fn(event, payload)); + } + } + } + }); + }); + + // debugAPI + wsServer.listenToChannel = (topic: Topic, fn: EventHandler) => { + if (!channels[topic]) { + channels[topic] = []; + } + channels[topic].push(fn); + }; + + return wsServer; +} + +export default createServer; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 6bc9fb6fd..bedc2b97d 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -69,6 +69,14 @@ type ServerOptions = { logger?: Logger; }; +// for now all I wanna do is say hello +const connectToLightning = (url: string, id: string) => { + let socket = new Socket(url /*,{params: {userToken: "123"}}*/); + socket.connect(); + const channel = socket.channel('worker'); + channel.push('hello'); +}; + function createServer(rtm: any, options: ServerOptions = {}) { const logger = options.logger || createMockLogger(); const port = options.port || 1234; @@ -104,7 +112,8 @@ function createServer(rtm: any, options: ServerOptions = {}) { if (options.lightning) { logger.log('Starting work loop at', options.lightning); - startWorkLoop(options.lightning, rtm.id, execute); + connectToLightning(options.lightning, rtm.id); + //startWorkLoop(options.lightning, rtm.id, execute); } else { logger.warn('No lightning URL provided'); } diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts new file mode 100644 index 000000000..b3f91e82e --- /dev/null +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -0,0 +1,50 @@ +import test from 'ava'; + +import phx from 'phoenix-channels'; +const { Socket } = phx; + +import createServer from '../../src/mock/socket-server'; + +let socket; +let server; + +test.beforeEach(() => { + server = createServer(); + + socket = new Socket('ws://localhost:8080'); + socket.connect(); +}); + +test.afterEach(() => { + server.close(); +}); + +test.serial('respond to connection join requests', async (t) => { + return new Promise((resolve) => { + const channel = socket.channel('x', {}); + + channel.join().receive('ok', (resp) => { + t.is(resp, 'ok'); + + channel.push('hello'); + resolve(); + }); + }); +}); + +test.serial('send a message', async (t) => { + return new Promise((resolve) => { + const channel = socket.channel('x', {}); + + server.listenToChannel('x', (event, payload) => { + t.is(event, 'hello'); + t.deepEqual(payload, { x: 1 }); + + resolve(); + }); + + channel.join(); + + channel.push('hello', { x: 1 }); + }); +}); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index b115cd564..8f2637e54 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import WebSocket, { WebSocketServer } from 'ws'; import createServer from '../src/server'; import createMockRTM from '../src/mock/runtime-manager'; @@ -6,18 +7,44 @@ import createMockRTM from '../src/mock/runtime-manager'; // Unit tests against the RTM web server // I don't think there will ever be much here because the server is mostly a pull +let rtm; let server; const url = 'http://localhost:7777'; test.beforeEach(() => { - const rtm = createMockRTM(); - server = createServer(rtm, { port: 7777 }); + rtm = createMockRTM(); }); -test('healthcheck', async (t) => { +test.afterEach(() => { + server.close(); // whatever +}); + +test.skip('healthcheck', async (t) => { + const server = createServer(rtm, { port: 7777 }); const result = await fetch(`${url}/healthcheck`); t.is(result.status, 200); const body = await result.text(); t.is(body, 'OK'); }); + +test('connects to websocket', (t) => { + let didSayHello; + + const wss = new WebSocketServer({ + port: 8080, + }); + wss.on('message', () => { + didSayHello = true; + }); + + rtm = createMockRTM(); + server = createServer(rtm, { + port: 7777, + lightning: 'ws://localhost:8080', + // TODO what if htere's some kind of onready hook? + // TODO also we'll need some utility like waitForEvent + }); + + t.true(didSayHello); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ab206d970..2ae0b8764 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,12 +106,9 @@ importers: integration-tests/cli/repo: dependencies: - '@openfn/language-common_1.10.3': - specifier: npm:@openfn/language-common@^1.10.3 - version: /@openfn/language-common@1.10.3 - '@openfn/language-http_5.0.2': - specifier: npm:@openfn/language-http@^5.0.2 - version: /@openfn/language-http@5.0.2 + '@openfn/language-common_1.11.0': + specifier: npm:@openfn/language-common@^1.11.0 + version: /@openfn/language-common@1.11.0 packages/cli: dependencies: @@ -396,6 +393,12 @@ importers: koa-logger: specifier: ^3.2.1 version: 3.2.1 + phoenix-channels: + specifier: ^1.0.0 + version: 1.0.0 + ws: + specifier: ^8.14.1 + version: 8.14.1 devDependencies: '@types/koa': specifier: ^2.13.5 @@ -1446,15 +1449,17 @@ packages: semver: 7.5.4 dev: true - /@openfn/language-common@1.10.3: - resolution: {integrity: sha512-abFOM/aj/L7qhiQEJyxX95HL2mARINNfXEt/SUcdMKcbENSbu+KRKGv7spBrn72PJ9pngo4/+OWxmQzzt4FEfQ==} + /@openfn/language-common@1.11.0: + resolution: {integrity: sha512-fd7d2ML03qNTAKu1PY/9zCV/TL3ZVsf70CJSpJUW3uloazlj/fDLRPmwr4Qzb8aYUB0qSGh1xY16s/e7N3NhOQ==} dependencies: + ajv: 8.12.0 axios: 1.1.3 csv-parse: 5.4.0 csvtojson: 2.0.10 date-fns: 2.30.0 jsonpath-plus: 4.0.0 lodash: 4.17.21 + undici: 5.24.0 transitivePeerDependencies: - debug dev: false @@ -1474,22 +1479,6 @@ packages: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] - /@openfn/language-http@5.0.2: - resolution: {integrity: sha512-jSfMcObunuxsbv/nc3HtOOurw1kv1cPf1NmytZxr4x9E+zZGynQjt4GXbNeQEEAtABX9mcsl3F1k2N+IMsyHpQ==} - dependencies: - '@openfn/language-common': 1.10.3 - cheerio: 1.0.0-rc.12 - cheerio-tableparser: 1.0.1 - csv-parse: 4.16.3 - fast-safe-stringify: 2.1.1 - form-data: 3.0.1 - lodash: 4.17.21 - request: 2.88.2 - tough-cookie: 4.1.3 - transitivePeerDependencies: - - debug - dev: false - /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1920,12 +1909,12 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 - /ajv@6.12.6: - resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + /ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} dependencies: fast-deep-equal: 3.1.3 - fast-json-stable-stringify: 2.1.0 - json-schema-traverse: 0.4.1 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 uri-js: 4.4.1 dev: false @@ -2064,17 +2053,6 @@ packages: resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} engines: {node: '>=12'} - /asn1@0.2.6: - resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} - dependencies: - safer-buffer: 2.1.2 - dev: false - - /assert-plus@1.0.0: - resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} - engines: {node: '>=0.8'} - dev: false - /assign-symbols@1.0.0: resolution: {integrity: sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==} engines: {node: '>=0.10.0'} @@ -2230,14 +2208,6 @@ packages: fast-glob: 3.3.1 dev: true - /aws-sign2@0.7.0: - resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} - dev: false - - /aws4@1.12.0: - resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} - dev: false - /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: @@ -2291,12 +2261,6 @@ packages: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} dev: true - /bcrypt-pbkdf@1.0.2: - resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} - dependencies: - tweetnacl: 0.14.5 - dev: false - /bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: true @@ -2348,10 +2312,6 @@ packages: /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} - /boolbase@1.0.0: - resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - dev: false - /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2418,6 +2378,14 @@ packages: ieee754: 1.2.1 dev: true + /bufferutil@4.0.7: + resolution: {integrity: sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.6.1 + dev: false + /builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} dependencies: @@ -2444,6 +2412,13 @@ packages: load-tsconfig: 0.2.5 dev: true + /busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + dependencies: + streamsearch: 1.1.0 + dev: false + /bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -2536,10 +2511,6 @@ packages: engines: {node: '>=6'} dev: true - /caseless@0.12.0: - resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} - dev: false - /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -2573,34 +2544,6 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} - /cheerio-select@2.1.0: - resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - dependencies: - boolbase: 1.0.0 - css-select: 5.1.0 - css-what: 6.1.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - dev: false - - /cheerio-tableparser@1.0.1: - resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} - dev: false - - /cheerio@1.0.0-rc.12: - resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} - engines: {node: '>= 6'} - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.1.0 - htmlparser2: 8.0.2 - parse5: 7.1.2 - parse5-htmlparser2-tree-adapter: 7.0.0 - dev: false - /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -2852,10 +2795,6 @@ packages: resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} dev: false - /core-util-is@1.0.2: - resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - dev: false - /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -2906,21 +2845,6 @@ packages: which: 2.0.2 dev: true - /css-select@5.1.0: - resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} - dependencies: - boolbase: 1.0.0 - css-what: 6.1.0 - domhandler: 5.0.3 - domutils: 3.1.0 - nth-check: 2.1.1 - dev: false - - /css-what@6.1.0: - resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} - engines: {node: '>= 6'} - dev: false - /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -2937,6 +2861,7 @@ packages: /csv-parse@4.16.3: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} + dev: true /csv-parse@5.4.0: resolution: {integrity: sha512-JiQosUWiOFgp4hQn0an+SBoV9IKdqzhROM0iiN4LB7UpfJBlsSJlWl9nq4zGgxgMAzHJ6V4t29VAVD+3+2NJAg==} @@ -2972,11 +2897,11 @@ packages: dependencies: array-find-index: 1.0.2 - /dashdash@1.14.1: - resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} - engines: {node: '>=0.10'} + /d@1.0.1: + resolution: {integrity: sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==} dependencies: - assert-plus: 1.0.0 + es5-ext: 0.10.62 + type: 1.2.0 dev: false /date-fns@2.30.0: @@ -3000,7 +2925,6 @@ packages: optional: true dependencies: ms: 2.0.0 - dev: true /debug@3.2.7(supports-color@5.5.0): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} @@ -3158,33 +3082,6 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true - /dom-serializer@2.0.0: - resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - entities: 4.5.0 - dev: false - - /domelementtype@2.3.0: - resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} - dev: false - - /domhandler@5.0.3: - resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} - engines: {node: '>= 4'} - dependencies: - domelementtype: 2.3.0 - dev: false - - /domutils@3.1.0: - resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} - dependencies: - dom-serializer: 2.0.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - dev: false - /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3208,13 +3105,6 @@ packages: /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - /ecc-jsbn@0.1.2: - resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} - dependencies: - jsbn: 0.1.1 - safer-buffer: 2.1.2 - dev: false - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3253,11 +3143,6 @@ packages: ansi-colors: 4.1.3 dev: true - /entities@4.5.0: - resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} - engines: {node: '>=0.12'} - dev: false - /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -3313,6 +3198,31 @@ packages: is-symbol: 1.0.4 dev: true + /es5-ext@0.10.62: + resolution: {integrity: sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==} + engines: {node: '>=0.10'} + requiresBuild: true + dependencies: + es6-iterator: 2.0.3 + es6-symbol: 3.1.3 + next-tick: 1.1.0 + dev: false + + /es6-iterator@2.0.3: + resolution: {integrity: sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==} + dependencies: + d: 1.0.1 + es5-ext: 0.10.62 + es6-symbol: 3.1.3 + dev: false + + /es6-symbol@3.1.3: + resolution: {integrity: sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==} + dependencies: + d: 1.0.1 + ext: 1.7.0 + dev: false + /esbuild-android-64@0.14.54: resolution: {integrity: sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==} engines: {node: '>=12'} @@ -3929,6 +3839,12 @@ packages: - supports-color dev: true + /ext@1.7.0: + resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} + dependencies: + type: 2.7.2 + dev: false + /extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} engines: {node: '>=0.10.0'} @@ -3944,10 +3860,6 @@ packages: is-extendable: 1.0.1 dev: true - /extend@3.0.2: - resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - dev: false - /extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} dev: true @@ -3976,11 +3888,6 @@ packages: - supports-color dev: true - /extsprintf@1.3.0: - resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} - engines: {'0': node >=0.6.0} - dev: false - /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false @@ -4017,10 +3924,6 @@ packages: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} dev: false - /fast-json-stable-stringify@2.1.0: - resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} - dev: false - /fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} dev: true @@ -4144,19 +4047,6 @@ packages: signal-exit: 4.0.2 dev: true - /forever-agent@0.6.1: - resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} - dev: false - - /form-data@2.3.3: - resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} - engines: {node: '>= 0.12'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -4166,15 +4056,6 @@ packages: mime-types: 2.1.35 dev: true - /form-data@3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4298,12 +4179,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /getpass@0.1.7: - resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} - dependencies: - assert-plus: 1.0.0 - dev: false - /glob-parent@3.1.0: resolution: {integrity: sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==} dependencies: @@ -4410,20 +4285,6 @@ packages: through2: 2.0.5 dev: true - /har-schema@2.0.0: - resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} - engines: {node: '>=4'} - dev: false - - /har-validator@5.1.5: - resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} - engines: {node: '>=6'} - deprecated: this library is no longer supported - dependencies: - ajv: 6.12.6 - har-schema: 2.0.0 - dev: false - /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -4509,15 +4370,6 @@ packages: lru-cache: 7.18.3 dev: true - /htmlparser2@8.0.2: - resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - entities: 4.5.0 - dev: false - /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4586,15 +4438,6 @@ packages: - supports-color dev: true - /http-signature@1.2.0: - resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} - engines: {node: '>=0.8', npm: '>=1.3.7'} - dependencies: - assert-plus: 1.0.0 - jsprim: 1.4.2 - sshpk: 1.17.0 - dev: false - /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -5041,10 +4884,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /isstream@0.1.2: - resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} - dev: false - /jackspeak@2.2.2: resolution: {integrity: sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==} engines: {node: '>=14'} @@ -5081,10 +4920,6 @@ packages: argparse: 2.0.1 dev: true - /jsbn@0.1.1: - resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} - dev: false - /json-diff@1.0.6: resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} hasBin: true @@ -5098,16 +4933,8 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true - /json-schema-traverse@0.4.1: - resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - dev: false - - /json-schema@0.4.0: - resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} - dev: false - - /json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} dev: false /jsonfile@4.0.0: @@ -5133,16 +4960,6 @@ packages: underscore: 1.12.1 dev: true - /jsprim@1.4.2: - resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} - engines: {node: '>=0.6.0'} - dependencies: - assert-plus: 1.0.0 - extsprintf: 1.3.0 - json-schema: 0.4.0 - verror: 1.10.0 - dev: false - /keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} @@ -5690,7 +5507,6 @@ packages: /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} - dev: true /ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} @@ -5745,6 +5561,10 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} + /next-tick@1.1.0: + resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} + dev: false + /no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} dependencies: @@ -5764,6 +5584,11 @@ packages: whatwg-url: 5.0.0 dev: false + /node-gyp-build@4.6.1: + resolution: {integrity: sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==} + hasBin: true + dev: false + /nodemon@2.0.19: resolution: {integrity: sha512-4pv1f2bMDj0Eeg/MhGqxrtveeQ5/G/UVe9iO6uTZzjnRluSA4PVWf8CW99LUPwGB3eNIA7zUFoP77YuI7hOc0A==} engines: {node: '>=8.10.0'} @@ -5862,16 +5687,6 @@ packages: path-key: 3.1.1 dev: true - /nth-check@2.1.1: - resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} - dependencies: - boolbase: 1.0.0 - dev: false - - /oauth-sign@0.9.0: - resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} - dev: false - /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6141,19 +5956,6 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} - /parse5-htmlparser2-tree-adapter@7.0.0: - resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} - dependencies: - domhandler: 5.0.3 - parse5: 7.1.2 - dev: false - - /parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - dependencies: - entities: 4.5.0 - dev: false - /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6230,8 +6032,12 @@ packages: through2: 2.0.5 dev: true - /performance-now@2.1.0: - resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} + /phoenix-channels@1.0.0: + resolution: {integrity: sha512-NVanumwkjzxUptGKBAMQ1W9njWrJom61rNpcTvSho9Hs441Lv8AJElXdkbycX9fFccc6OJViVLhDL0L5U/HqMg==} + dependencies: + websocket: 1.0.34 + transitivePeerDependencies: + - supports-color dev: false /picocolors@1.0.0: @@ -6464,10 +6270,6 @@ packages: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} dev: true - /psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - dev: false - /pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -6498,15 +6300,6 @@ packages: side-channel: 1.0.4 dev: false - /qs@6.5.3: - resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} - engines: {node: '>=0.6'} - dev: false - - /querystringify@2.2.0: - resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} - dev: false - /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6691,45 +6484,19 @@ packages: engines: {node: '>=0.10'} dev: true - /request@2.88.2: - resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} - engines: {node: '>= 6'} - deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 - dependencies: - aws-sign2: 0.7.0 - aws4: 1.12.0 - caseless: 0.12.0 - combined-stream: 1.0.8 - extend: 3.0.2 - forever-agent: 0.6.1 - form-data: 2.3.3 - har-validator: 5.1.5 - http-signature: 1.2.0 - is-typedarray: 1.0.0 - isstream: 0.1.2 - json-stringify-safe: 5.0.1 - mime-types: 2.1.35 - oauth-sign: 0.9.0 - performance-now: 2.1.0 - qs: 6.5.3 - safe-buffer: 5.2.1 - tough-cookie: 2.5.0 - tunnel-agent: 0.6.0 - uuid: 3.4.0 - dev: false - /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true - /requires-port@1.0.0: - resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} - dev: false - /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -7148,22 +6915,6 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - /sshpk@1.17.0: - resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==} - engines: {node: '>=0.10.0'} - hasBin: true - dependencies: - asn1: 0.2.6 - assert-plus: 1.0.0 - bcrypt-pbkdf: 1.0.2 - dashdash: 1.14.1 - ecc-jsbn: 0.1.2 - getpass: 0.1.7 - jsbn: 0.1.1 - safer-buffer: 2.1.2 - tweetnacl: 0.14.5 - dev: false - /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7215,6 +6966,11 @@ packages: mixme: 0.5.4 dev: true + /streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + dev: false + /streamx@2.13.0: resolution: {integrity: sha512-9jD4uoX0juNSIcv4PazT+97FpM4Mww3cp7PM23HRTLANhgb7K7n1mB45guH/kT5F4enl04kApOM3EeoUXSPfvw==} dependencies: @@ -7495,24 +7251,6 @@ packages: nopt: 1.0.10 dev: true - /tough-cookie@2.5.0: - resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} - engines: {node: '>=0.8'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - dev: false - - /tough-cookie@4.1.3: - resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} - engines: {node: '>=6'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - universalify: 0.2.0 - url-parse: 1.5.10 - dev: false - /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false @@ -7772,16 +7510,6 @@ packages: yargs: 17.7.2 dev: true - /tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - dependencies: - safe-buffer: 5.2.1 - dev: false - - /tweetnacl@0.14.5: - resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} - dev: false - /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -7815,6 +7543,20 @@ packages: mime-types: 2.1.35 dev: false + /type@1.2.0: + resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==} + dev: false + + /type@2.7.2: + resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==} + dev: false + + /typedarray-to-buffer@3.1.5: + resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} + dependencies: + is-typedarray: 1.0.0 + dev: false + /typescript@4.6.4: resolution: {integrity: sha512-9ia/jWHIEbo49HfjrLGfKbZSuWo9iTMwXO+Ca3pRsSpbsMbc7/IU8NKdCZVRRBafVPGnoJeFL76ZOAA84I9fEg==} engines: {node: '>=4.2.0'} @@ -7860,6 +7602,13 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true + /undici@5.24.0: + resolution: {integrity: sha512-OKlckxBjFl0oXxcj9FU6oB8fDAaiRUq+D8jrFWGmOfI/gIyjk/IeS75LMzgYKUaeHzLUcYvf9bbJGSrUwTfwwQ==} + engines: {node: '>=14.0'} + dependencies: + busboy: 1.6.0 + dev: false + /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'} @@ -7889,11 +7638,6 @@ packages: engines: {node: '>= 4.0.0'} dev: true - /universalify@0.2.0: - resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} - engines: {node: '>= 4.0.0'} - dev: false - /unix-crypt-td-js@1.1.4: resolution: {integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==} dev: true @@ -7931,18 +7675,19 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false - /url-parse@1.5.10: - resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - dev: false - /use@3.1.1: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} dev: true + /utf-8-validate@5.0.10: + resolution: {integrity: sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.6.1 + dev: false + /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} dev: true @@ -7956,6 +7701,7 @@ packages: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. hasBin: true + dev: true /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -7978,15 +7724,6 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - /verror@1.10.0: - resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} - engines: {'0': node >=0.6.0} - dependencies: - assert-plus: 1.0.0 - core-util-is: 1.0.2 - extsprintf: 1.3.0 - dev: false - /wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} dependencies: @@ -8015,6 +7752,20 @@ packages: engines: {node: '>=0.8.0'} dev: true + /websocket@1.0.34: + resolution: {integrity: sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ==} + engines: {node: '>=4.0.0'} + dependencies: + bufferutil: 4.0.7 + debug: 2.6.9 + es5-ext: 0.10.62 + typedarray-to-buffer: 3.1.5 + utf-8-validate: 5.0.10 + yaeti: 0.0.6 + transitivePeerDependencies: + - supports-color + dev: false + /well-known-symbols@2.0.0: resolution: {integrity: sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==} engines: {node: '>=6'} @@ -8119,6 +7870,19 @@ packages: imurmurhash: 0.1.4 signal-exit: 4.0.2 + /ws@8.14.1: + resolution: {integrity: sha512-4OOseMUq8AzRBI/7SLMUwO+FEDnguetSk7KMb1sHwvF2w2Wv5Hoj0nlifx8vtGsftE/jWHojPy8sMMzYLJ2G/A==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -8132,6 +7896,11 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} + /yaeti@0.0.6: + resolution: {integrity: sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==} + engines: {node: '>=0.10.32'} + dev: false + /yallist@2.1.2: resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} dev: true From 7254a0235f8e609db32e6fde97401db09852cb14 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 15 Sep 2023 15:43:23 +0200 Subject: [PATCH 060/232] rtm: more unit tests on sokcet mock --- packages/rtm-server/src/mock/socket-server.ts | 31 +++++++-- .../test/mock/socket-server.test.ts | 68 +++++++++++++++++++ 2 files changed, 92 insertions(+), 7 deletions(-) diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/socket-server.ts index 369895e33..6156ef6c8 100644 --- a/packages/rtm-server/src/mock/socket-server.ts +++ b/packages/rtm-server/src/mock/socket-server.ts @@ -17,15 +17,12 @@ type PhoenixEvent = { type EventHandler = (event: string, payload: any) => void; function createServer() { - const channels: Record = {}; + const channels: Record> = {}; const wsServer = new WebSocketServer({ port: 8080, }); - // util to send a response to a particular topic - const reply = () => {}; - const events = { // When joining a channel, we need to send a chan_reply_{ref} message back to the socket phx_join: (ws, { topic, ref }) => { @@ -55,7 +52,9 @@ function createServer() { } else { // handle custom/user events if (channels[topic]) { - channels[topic].forEach((fn) => fn(event, payload)); + channels[topic].forEach((fn) => { + fn(event, payload)} + ); } } } @@ -65,11 +64,29 @@ function createServer() { // debugAPI wsServer.listenToChannel = (topic: Topic, fn: EventHandler) => { if (!channels[topic]) { - channels[topic] = []; + channels[topic] = new Set() } - channels[topic].push(fn); + + channels[topic].add(fn); + + return { + unsubscribe: () => { + channels[topic].delete(fn) + } + }; }; + wsServer.waitForMessage = (topic: Topic, event: string) => { + return new Promise((resolve) => { + const listener = wsServer.listenToChannel(topic, (e: string, payload: any) => { + if (e === event) { + listener.unsubscribe(); + resolve(payload) + } + }) + }) + } + return wsServer; } diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index b3f91e82e..6e2f0ee40 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -8,6 +8,10 @@ import createServer from '../../src/mock/socket-server'; let socket; let server; +const wait = (duration = 10) => new Promise(resolve => { + setTimeout(resolve, duration) +}) + test.beforeEach(() => { server = createServer(); @@ -48,3 +52,67 @@ test.serial('send a message', async (t) => { channel.push('hello', { x: 1 }); }); }); + +test.serial('send a message only to one channel', async (t) => { + let didCallX = false; + let didCallY = false; + + const x = socket.channel('x', {}); + x.join(); + + const y = socket.channel('y', {}); + y.join(); + + server.listenToChannel('x', () => { + didCallX = true + }); + server.listenToChannel('y', () => { + didCallY = true + }); + + + x.push('hello', { x: 1 }); + + await wait() + + t.true(didCallX) + t.false(didCallY) +}); + +test.serial('unsubscribe', (t) => { + return new Promise(async (resolve) => { + let count = 0; + + const channel = socket.channel('x', {}); + channel.join(); + + const listener = server.listenToChannel('x', () => { + count++; + }); + + channel.push('hello', { x: 1 }); + await wait(100) + + t.is(count, 1); + + listener.unsubscribe(); + + channel.push('hello', { x: 1 }); + await wait() + + t.is(count, 1); + + resolve(); + }); +}); + + +test.serial('wait for message', async (t) => { + const channel = socket.channel('x', {}); + channel.join(); + + channel.push('hello', { x: 1 }); + + const result = await server.waitForMessage('x', 'hello'); + t.truthy(result) +}); From 089d34533d771e8f80d2147f25765c45c9362ca1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 18 Sep 2023 10:34:11 +0200 Subject: [PATCH 061/232] rtm: add some events --- packages/rtm-server/package.json | 2 ++ packages/rtm-server/src/events.ts | 19 +++++++++++++ .../rtm-server/src/mock/lightning/server.ts | 19 +++++++++++-- .../rtm-server/test/mock/lightning.test.ts | 27 +++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 packages/rtm-server/src/events.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index c9d20f73e..46b3215a8 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -35,6 +35,8 @@ "@types/node": "^18.15.3", "@types/yargs": "^17.0.12", "ava": "5.1.0", + "koa-route": "^3.2.0", + "koa-websocket": "^7.0.0", "nodemon": "3.0.1", "ts-node": "^10.9.1", "tslib": "^2.4.0", diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts new file mode 100644 index 000000000..1b108c55d --- /dev/null +++ b/packages/rtm-server/src/events.ts @@ -0,0 +1,19 @@ +// track socket event names as constants to keep refactoring easier + +const CLAIM = 'attempt:claim'; + +// TODO does each worker connect to its own channel, ensuring a private claim steeam? +// or is there a shared Workers channel + +// claim reply needs to include the id of the server and the attempt +const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: 'a1' } + + +// All attempt events are in a dedicated channel for that event + +const ATTEMPT_START = 'attempt:start' // attemptId, timestamp +const ATTEMPT_COMPLETE = 'attempt:complete' // attemptId, timestamp, result, stats +const ATTEMPT_LOG = 'attempt:complete' // level, namespace (job,runtime,adaptor), message, time + +// this should not happen - this is "could not execute" rather than "complete with errors" +const ATTEMPT_ERROR = 'attempt:error' \ No newline at end of file diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 958d290f4..84803be34 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -2,8 +2,11 @@ import { EventEmitter } from 'node:events'; import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; +import websockify from 'koa-websocket'; +import route from 'koa-route'; import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; +import createServer from '../socket-server'; import createAPI from './api'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; @@ -28,7 +31,7 @@ export type LightningOptions = { const createLightningServer = (options: LightningOptions = {}) => { const logger = options.logger || createMockLogger(); - // App state + // App state websockify = require('koa-websocket'); const state = { credentials: {}, attempts: [], @@ -41,9 +44,21 @@ const createLightningServer = (options: LightningOptions = {}) => { } as ServerState; // Server setup - const app = new Koa(); + const app = websockify(new Koa()); app.use(bodyParser()); + // Using routes + app.ws.use(route.all('/test/:id', function (ctx) { + // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. + // the websocket is added to the context on `ctx.websocket`. + ctx.websocket.send('Hello World'); + ctx.websocket.on('message', function(message) { + // do something with the message from client + console.log(message); + }); + })); + + const klogger = koaLogger((str) => logger.debug(str)); app.use(klogger); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 517fab36d..4a03b60bd 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -3,6 +3,9 @@ import { attempts } from '../../src/mock/data'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; import { createMockLogger } from '@openfn/logger'; +import phx from 'phoenix-channels'; +const { Socket } = phx; + const baseUrl = `http://localhost:8888${API_PREFIX}`; let server; @@ -32,6 +35,30 @@ const post = (path: string, data: any) => const attempt1 = attempts()['attempt-1']; +test.only('provide a websocket at /websocket', (t) => { + return new Promise((done) => { + const socket = new Socket(`ws://localhost:4000/socket`); + + socket.connect() + // .on('ok', () => { + // done(); + // }) + console.log(socket.connectionState()) + + t.is(socket.connectionState(), 'ok') + done(); + }); +}); + +// respond to a claim request with an id +// uh does this stuff make any sense in the socket model? + +// create a channel for an attempt + + + + + test.serial( 'GET /credential - return 404 if no credential found', async (t) => { From 9b356942074f0273414beb6bdbcfeca46ad86553 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 18 Sep 2023 15:15:55 +0100 Subject: [PATCH 062/232] rtm: get basic sockets working in lightning --- packages/rtm-server/socket3.js | 66 +++++ packages/rtm-server/socket4.js | 40 +++ packages/rtm-server/src/mock/lightning/api.ts | 30 +- .../rtm-server/src/mock/lightning/server.ts | 50 +++- packages/rtm-server/src/mock/socket-server.ts | 53 ++-- .../rtm-server/test/mock/lightning.test.ts | 269 +++++++++++------- .../test/mock/socket-server.test.ts | 35 ++- pnpm-lock.yaml | 43 ++- 8 files changed, 426 insertions(+), 160 deletions(-) create mode 100644 packages/rtm-server/socket3.js create mode 100644 packages/rtm-server/socket4.js diff --git a/packages/rtm-server/socket3.js b/packages/rtm-server/socket3.js new file mode 100644 index 000000000..2436eddeb --- /dev/null +++ b/packages/rtm-server/socket3.js @@ -0,0 +1,66 @@ +// create a koa ws server +import Koa from 'koa'; +import route from 'koa-route'; +import websockify from 'koa-websocket'; +import Socket from 'ws'; + +// learnings: route is .all(path, fn) +// socket takes on 'message' + +const app = websockify(new Koa()); + +console.log(app.ws); + +app.ws.server.on('connection', () => { + console.log(' >> connect'); +}); + +// Regular middleware +// Note it's app.ws.use and not app.use +app.ws.use( + route.all('/jam', (ctx, next) => { + console.log('>> jam'); + + // I need this connection from the server, not the socket + // koa-ws hides that from me + + ctx.websocket.on('message', (m) => { + const x = m.toString(); + if (x === 'ping') { + console.log('received'); + ctx.websocket.send('pong'); + } + }); + + // return `next` to pass the context (ctx) on to the next ws middleware + return next(ctx); + }) +); + +// // Using routes +// app.ws.use(route.all('/test/:id', function (ctx) { +// // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. +// // the websocket is added to the context on `ctx.websocket`. +// ctx.websocket.send('Hello World'); +// ctx.websocket.on('message', function(message) { +// // do something with the message from client +// console.log(message); +// }); +// })); + +app.listen(3333); + +const s = new Socket('ws://localhost:3333/jam'); + +s.on('open', () => { + console.log('pinging...'); + s.send('ping'); +}); + +s.on('message', (m) => { + const message = m.toString(); + if (message === 'pong') { + console.log('pong!'); + process.exit(0); + } +}); diff --git a/packages/rtm-server/socket4.js b/packages/rtm-server/socket4.js new file mode 100644 index 000000000..3b46a5123 --- /dev/null +++ b/packages/rtm-server/socket4.js @@ -0,0 +1,40 @@ +// This attempt builds my own websocket into koa, only available at one path +// Then I can use the connection handler myself to plug in my phoenix mock +import Koa from 'koa'; +import Socket, { WebSocketServer } from 'ws'; + +const app = new Koa(); + +const server = app.listen(3333); + +const wss = new WebSocketServer({ + server, + path: '/jam', +}); + +wss.on('connection', (socket, req) => { + console.log('>> connection'); + socket.on('message', (m) => { + console.log(m); + const x = m.toString(); + if (x === 'ping') { + console.log('received'); + socket.send('pong'); + } + }); +}); + +const s = new Socket('ws://localhost:3333/jam'); + +s.on('open', () => { + console.log('pinging...'); + s.send('ping'); +}); + +s.on('message', (m) => { + const message = m.toString(); + if (message === 'pong') { + console.log('pong!'); + process.exit(0); + } +}); diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 9c80b4631..346424385 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -1,4 +1,5 @@ import Router from '@koa/router'; +import Socket, { WebSocketServer } from 'ws'; import { unimplemented, createListNextJob, @@ -11,6 +12,8 @@ import type { ServerState } from './server'; import { API_PREFIX } from './server'; +import createPheonixMockSocketServer from '../socket-server'; + interface RTMBody { rtm_id: string; } @@ -26,7 +29,23 @@ export interface AttemptCompleteBody extends RTMBody { // can I even implement this in JS? Not with pheonix anyway. hmm. // dead at the first hurdle really. // what if I do the server side mock in koa, can I use the pheonix client to connect? -const newApi = () => { +export const createNewAPI = (state: ServerState, path: string, server) => { + // set up a websocket server to listen to connections + const wss = new WebSocketServer({ + server, + path, + }); + + // pass that through to the phoenix mock + createPheonixMockSocketServer({ server: wss }); + console.log('mock created'); + + // then do something clever to map events + // server.on({ + // 'attempt:claim': noop, + // 'attempt:start': noop, + // }) + const noop = () => {}; // This may actually get split into a server bit and a an attempts bit, reflecting the different channels @@ -46,9 +65,14 @@ const newApi = () => { 'run:log': noop, }; - const handleEvent = (name) => {}; + // const handleEvent = (name) => {}; - return handleEvent; + // return handleEvent; + + return (ctx) => { + // what does this actually do? + console.log(' >> ', ctx); + }; }; // Note that this API is hosted at api/1 diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 84803be34..677f2e2ad 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -7,7 +7,7 @@ import route from 'koa-route'; import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; import createServer from '../socket-server'; -import createAPI from './api'; +import createAPI, { createNewAPI } from './api'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; @@ -44,20 +44,44 @@ const createLightningServer = (options: LightningOptions = {}) => { } as ServerState; // Server setup - const app = websockify(new Koa()); + // this seems to setup websockets to work at any path + // Maybe that's fine for the mock? Kind wierd + // mind you I still don't get a connect event + // const app = websockify(new Koa()); + const app = new Koa(); app.use(bodyParser()); - // Using routes - app.ws.use(route.all('/test/:id', function (ctx) { - // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. - // the websocket is added to the context on `ctx.websocket`. - ctx.websocket.send('Hello World'); - ctx.websocket.on('message', function(message) { - // do something with the message from client - console.log(message); - }); - })); + // this router but doesn't really seem to work + // app.use(route.all('/websocket', createNewAPI(state, options.port || 8888))); + // Using routes (seems to not work - at least the api doesn't get) + // app.ws.use( + // route.all('/websocket', (ctx) => createNewAPI(state, ctx.websocket)) + // ); + + // this probaably isn;t right because it'll create a new API object + /// for each request + + const server = app.listen(options.port || 8888); + + createNewAPI(state, '/websocket', server); + + // I really don't think this should be hard + // we connect to a socket sitting at /socket + // then we sub to events on both sides + + // app.ws.use( + // route.all('/websocket', function (ctx) { + // console.log(' >> WEBSOCKET '); + // // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. + // // the websocket is added to the context on `ctx.websocket`. + // ctx.websocket.send('Hello World'); + // ctx.websocket.on('message', function (message) { + // // do something with the message from client + // console.log(message); + // }); + // }) + // ); const klogger = koaLogger((str) => logger.debug(str)); app.use(klogger); @@ -66,8 +90,8 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(createAPI(state)); app.use(createDevAPI(app as any, state, logger)); - const server = app.listen(options.port || 8888); app.destroy = () => { + console.log('close'); server.close(); }; diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/socket-server.ts index 6156ef6c8..b8acb5239 100644 --- a/packages/rtm-server/src/mock/socket-server.ts +++ b/packages/rtm-server/src/mock/socket-server.ts @@ -16,16 +16,32 @@ type PhoenixEvent = { type EventHandler = (event: string, payload: any) => void; -function createServer() { +function createServer({ port = 8080, server } = {}) { + // console.log('ws listening on', port); const channels: Record> = {}; - const wsServer = new WebSocketServer({ - port: 8080, - }); + const wsServer = server; + // server || + // new WebSocketServer({ + // port, + // }); const events = { + // testing (TODO shouldn't this be in a specific channel?) + ping: (ws, { topic, ref }) => { + console.log(' >> ping'); + ws.send( + JSON.stringify({ + topic, + ref, + event: 'pong', + payload: {}, + }) + ); + }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket phx_join: (ws, { topic, ref }) => { + console.log('-- join --'); ws.send( JSON.stringify({ // here is the magic reply event @@ -40,7 +56,9 @@ function createServer() { }; wsServer.on('connection', function (ws: WS) { + console.log(' >> connect'); ws.on('message', function (data: string) { + console.log(' >> message'); const evt = JSON.parse(data) as PhoenixEvent; if (evt.topic) { // phx sends this info in each message @@ -53,8 +71,8 @@ function createServer() { // handle custom/user events if (channels[topic]) { channels[topic].forEach((fn) => { - fn(event, payload)} - ); + fn(event, payload); + }); } } } @@ -64,28 +82,31 @@ function createServer() { // debugAPI wsServer.listenToChannel = (topic: Topic, fn: EventHandler) => { if (!channels[topic]) { - channels[topic] = new Set() + channels[topic] = new Set(); } channels[topic].add(fn); return { unsubscribe: () => { - channels[topic].delete(fn) - } + channels[topic].delete(fn); + }, }; }; wsServer.waitForMessage = (topic: Topic, event: string) => { return new Promise((resolve) => { - const listener = wsServer.listenToChannel(topic, (e: string, payload: any) => { - if (e === event) { - listener.unsubscribe(); - resolve(payload) + const listener = wsServer.listenToChannel( + topic, + (e: string, payload: any) => { + if (e === event) { + listener.unsubscribe(); + resolve(payload); + } } - }) - }) - } + ); + }); + }; return wsServer; } diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 4a03b60bd..bed3e3bca 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -6,12 +6,17 @@ import { createMockLogger } from '@openfn/logger'; import phx from 'phoenix-channels'; const { Socket } = phx; -const baseUrl = `http://localhost:8888${API_PREFIX}`; +const baseUrl = `http://localhost:7777${API_PREFIX}`; + +const sleep = (duration = 10) => + new Promise((resolve) => { + setTimeout(resolve, duration); + }); let server; test.before(() => { - server = createLightningServer({ port: 8888 }); + server = createLightningServer({ port: 7777 }); }); test.afterEach(() => { @@ -35,31 +40,65 @@ const post = (path: string, data: any) => const attempt1 = attempts()['attempt-1']; -test.only('provide a websocket at /websocket', (t) => { - return new Promise((done) => { - const socket = new Socket(`ws://localhost:4000/socket`); +test.serial.skip('provide a phoenix websocket at /websocket', (t) => { + return new Promise(async (done) => { + const socket = new Socket(`ws://localhost:7777/websocket`); - socket.connect() - // .on('ok', () => { - // done(); - // }) - console.log(socket.connectionState()) - - t.is(socket.connectionState(), 'ok') + socket.connect(); + await sleep(); // TODO untidy + t.is(socket.connectionState(), 'open'); done(); }); }); -// respond to a claim request with an id -// uh does this stuff make any sense in the socket model? +test.serial('respond to connection join requests', (t) => { + return new Promise(async (done) => { + const socket = new Socket(`ws://localhost:7777`); + //const socket = new Socket(`ws://localhost:7777/websocket`); // TODO this breaks??? -// create a channel for an attempt + socket.connect(); + const channel = socket.channel('x', {}); + channel.join().receive('ok', (resp) => { + t.is(resp, 'ok'); + done(); + }); + }); +}); +// Thinking a bit about messaging flow +// a) it's not working (no connect, no join) +// b) the way this is written is awful +test.serial('get a reply to a ping event', (t) => { + return new Promise(async (done) => { + let didGetReply = false; + const socket = new Socket(`ws://localhost:7777`); + + socket.connect(); + // join the worker pool + const channel = socket.channel('workers', {}); + channel.join().receive('ok', () => { + // should get a response + channel.on('pong', (payload) => { + console.log('[ping] reply', payload); + didGetReply = true; + + t.true(didGetReply); + done(); + }); + + // TODO explicit test that the backing socket got this event? + channel.push('ping'); + }); + }); +}); +// respond to a claim request with an id +// uh does this stuff make any sense in the socket model? +// create a channel for an attempt -test.serial( +test.serial.skip( 'GET /credential - return 404 if no credential found', async (t) => { const res = await get('credential/x'); @@ -67,7 +106,7 @@ test.serial( } ); -test.serial('GET /credential - return a credential', async (t) => { +test.serial.skip('GET /credential - return a credential', async (t) => { server.addCredential('a', { user: 'johnny', password: 'cash' }); const res = await get('credential/a'); @@ -79,7 +118,7 @@ test.serial('GET /credential - return a credential', async (t) => { t.is(job.password, 'cash'); }); -test.serial( +test.serial.skip( 'POST /attempts/next - return 204 and no body for an empty queue', async (t) => { t.is(server.getQueueLength(), 0); @@ -89,32 +128,38 @@ test.serial( } ); -test.serial('POST /attempts/next - return 400 if no id provided', async (t) => { - const res = await post('attempts/next', {}); - t.is(res.status, 400); -}); +test.serial.skip( + 'POST /attempts/next - return 400 if no id provided', + async (t) => { + const res = await post('attempts/next', {}); + t.is(res.status, 400); + } +); -test.serial('GET /attempts/next - return 200 with a workflow', async (t) => { - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 1); +test.serial.skip( + 'GET /attempts/next - return 200 with a workflow', + async (t) => { + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 1); - const res = await post('attempts/next', { rtm_id: 'rtm' }); - const result = await res.json(); - t.is(res.status, 200); + const res = await post('attempts/next', { rtm_id: 'rtm' }); + const result = await res.json(); + t.is(res.status, 200); - t.truthy(result); - t.true(Array.isArray(result)); - t.is(result.length, 1); + t.truthy(result); + t.true(Array.isArray(result)); + t.is(result.length, 1); - // not interested in testing much against the attempt structure at this stage - const [attempt] = result; - t.is(attempt.id, 'attempt-1'); - t.true(Array.isArray(attempt.plan)); + // not interested in testing much against the attempt structure at this stage + const [attempt] = result; + t.is(attempt.id, 'attempt-1'); + t.true(Array.isArray(attempt.plan)); - t.is(server.getQueueLength(), 0); -}); + t.is(server.getQueueLength(), 0); + } +); -test.serial( +test.serial.skip( 'GET /attempts/next - return 200 with a workflow with an inline item', async (t) => { server.enqueueAttempt({ id: 'abc' }); @@ -135,24 +180,27 @@ test.serial( } ); -test.serial('GET /attempts/next - return 200 with 2 workflows', async (t) => { - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 3); +test.serial.skip( + 'GET /attempts/next - return 200 with 2 workflows', + async (t) => { + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 3); - const res = await post('attempts/next?count=2', { rtm_id: 'rtm' }); - t.is(res.status, 200); + const res = await post('attempts/next?count=2', { rtm_id: 'rtm' }); + t.is(res.status, 200); - const result = await res.json(); - t.truthy(result); - t.true(Array.isArray(result)); - t.is(result.length, 2); + const result = await res.json(); + t.truthy(result); + t.true(Array.isArray(result)); + t.is(result.length, 2); - t.is(server.getQueueLength(), 1); -}); + t.is(server.getQueueLength(), 1); + } +); -test.serial( +test.serial.skip( 'POST /attempts/next - clear the queue after a request', async (t) => { server.enqueueAttempt(attempt1); @@ -167,7 +215,7 @@ test.serial( } ); -test.serial('POST /attempts/log - should return 200', async (t) => { +test.serial.skip('POST /attempts/log - should return 200', async (t) => { server.enqueueAttempt(attempt1); const { status } = await post('attempts/log/attempt-1', { rtm_id: 'rtm', @@ -176,7 +224,7 @@ test.serial('POST /attempts/log - should return 200', async (t) => { t.is(status, 200); }); -test.serial( +test.serial.skip( 'POST /attempts/log - should return 400 if no rtm_id', async (t) => { const { status } = await post('attempts/log/attempt-1', { @@ -187,29 +235,32 @@ test.serial( } ); -test.serial('POST /attempts/log - should echo to event emitter', async (t) => { - server.enqueueAttempt(attempt1); - let evt; - let didCall = false; +test.serial.skip( + 'POST /attempts/log - should echo to event emitter', + async (t) => { + server.enqueueAttempt(attempt1); + let evt; + let didCall = false; - server.once('log', (e) => { - didCall = true; - evt = e; - }); + server.once('log', (e) => { + didCall = true; + evt = e; + }); - const { status } = await post('attempts/log/attempt-1', { - rtm_id: 'rtm', - logs: [{ message: 'hello world' }], - }); - t.is(status, 200); - t.true(didCall); + const { status } = await post('attempts/log/attempt-1', { + rtm_id: 'rtm', + logs: [{ message: 'hello world' }], + }); + t.is(status, 200); + t.true(didCall); - t.truthy(evt); - t.is(evt.id, 'attempt-1'); - t.deepEqual(evt.logs, [{ message: 'hello world' }]); -}); + t.truthy(evt); + t.is(evt.id, 'attempt-1'); + t.deepEqual(evt.logs, [{ message: 'hello world' }]); + } +); -test.serial('POST /attempts/complete - return final state', async (t) => { +test.serial.skip('POST /attempts/complete - return final state', async (t) => { server.enqueueAttempt(attempt1); const { status } = await post('attempts/complete/attempt-1', { rtm_id: 'rtm', @@ -222,18 +273,21 @@ test.serial('POST /attempts/complete - return final state', async (t) => { t.deepEqual(result, { x: 10 }); }); -test.serial('POST /attempts/complete - reject if unknown rtm', async (t) => { - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - x: 10, - }, - }); - t.is(status, 400); - t.falsy(server.getResult('attempt-1')); -}); +test.serial.skip( + 'POST /attempts/complete - reject if unknown rtm', + async (t) => { + const { status } = await post('attempts/complete/attempt-1', { + rtm_id: 'rtm', + state: { + x: 10, + }, + }); + t.is(status, 400); + t.falsy(server.getResult('attempt-1')); + } +); -test.serial( +test.serial.skip( 'POST /attempts/complete - reject if unknown workflow', async (t) => { server.enqueueAttempt({ id: 'b' }, 'rtm'); @@ -250,31 +304,34 @@ test.serial( } ); -test.serial('POST /attempts/complete - echo to event emitter', async (t) => { - server.enqueueAttempt(attempt1); - let evt; - let didCall = false; +test.serial.skip( + 'POST /attempts/complete - echo to event emitter', + async (t) => { + server.enqueueAttempt(attempt1); + let evt; + let didCall = false; - server.once('attempt-complete', (e) => { - didCall = true; - evt = e; - }); + server.once('attempt-complete', (e) => { + didCall = true; + evt = e; + }); - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - data: { - answer: 42, + const { status } = await post('attempts/complete/attempt-1', { + rtm_id: 'rtm', + state: { + data: { + answer: 42, + }, }, - }, - }); - t.is(status, 200); - t.true(didCall); + }); + t.is(status, 200); + t.true(didCall); - t.truthy(evt); - t.is(evt.rtm_id, 'rtm'); - t.is(evt.workflow_id, 'attempt-1'); - t.deepEqual(evt.state, { data: { answer: 42 } }); -}); + t.truthy(evt); + t.is(evt.rtm_id, 'rtm'); + t.is(evt.workflow_id, 'attempt-1'); + t.deepEqual(evt.state, { data: { answer: 42 } }); + } +); // test lightning should get the finished state through a helper API diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index 6e2f0ee40..ebc06dbd7 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -8,9 +8,10 @@ import createServer from '../../src/mock/socket-server'; let socket; let server; -const wait = (duration = 10) => new Promise(resolve => { - setTimeout(resolve, duration) -}) +const wait = (duration = 10) => + new Promise((resolve) => { + setTimeout(resolve, duration); + }); test.beforeEach(() => { server = createServer(); @@ -64,25 +65,24 @@ test.serial('send a message only to one channel', async (t) => { y.join(); server.listenToChannel('x', () => { - didCallX = true + didCallX = true; }); server.listenToChannel('y', () => { - didCallY = true + didCallY = true; }); - x.push('hello', { x: 1 }); - await wait() + await wait(); - t.true(didCallX) - t.false(didCallY) + t.true(didCallX); + t.false(didCallY); }); test.serial('unsubscribe', (t) => { return new Promise(async (resolve) => { let count = 0; - + const channel = socket.channel('x', {}); channel.join(); @@ -91,14 +91,14 @@ test.serial('unsubscribe', (t) => { }); channel.push('hello', { x: 1 }); - await wait(100) + await wait(100); t.is(count, 1); listener.unsubscribe(); channel.push('hello', { x: 1 }); - await wait() + await wait(); t.is(count, 1); @@ -106,13 +106,12 @@ test.serial('unsubscribe', (t) => { }); }); - test.serial('wait for message', async (t) => { - const channel = socket.channel('x', {}); - channel.join(); + const channel = socket.channel('x', {}); + channel.join(); - channel.push('hello', { x: 1 }); + channel.push('hello', { x: 1 }); - const result = await server.waitForMessage('x', 'hello'); - t.truthy(result) + const result = await server.waitForMessage('x', 'hello'); + t.truthy(result); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2ae0b8764..2352ed1e4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -418,6 +418,12 @@ importers: ava: specifier: 5.1.0 version: 5.1.0 + koa-route: + specifier: ^3.2.0 + version: 3.2.0 + koa-websocket: + specifier: ^7.0.0 + version: 7.0.0 nodemon: specifier: 3.0.1 version: 3.0.1 @@ -2679,7 +2685,6 @@ packages: /co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} - dev: false /code-excerpt@4.0.0: resolution: {integrity: sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==} @@ -4864,6 +4869,10 @@ packages: engines: {node: '>=4'} dev: true + /isarray@0.0.1: + resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} + dev: true + /isarray@1.0.0: resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} dev: true @@ -5006,7 +5015,6 @@ packages: /koa-compose@4.1.0: resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} - dev: false /koa-convert@2.0.0: resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} @@ -5026,6 +5034,29 @@ packages: passthrough-counter: 1.0.0 dev: false + /koa-route@3.2.0: + resolution: {integrity: sha512-8FsuWw/L+CUWJfpgN6vrlYUDNTheEinG8Zkm97GyuLJNyWjCVUs9p10Ih3jTIWwmDVQcz6827l0RKadAS5ibqA==} + dependencies: + debug: 4.3.4 + methods: 1.1.2 + path-to-regexp: 1.8.0 + transitivePeerDependencies: + - supports-color + dev: true + + /koa-websocket@7.0.0: + resolution: {integrity: sha512-MsHUFKqA4+j+0dpPKWtsvZfnpQ1NcgF+AaTZQZ4B3Xj/cWK31qqmKx9HnA5Gw1LV2aIDzqwy0IDBsZYRurTUAg==} + dependencies: + co: 4.6.0 + debug: 4.3.4 + koa-compose: 4.1.0 + ws: 8.14.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + dev: true + /koa@2.13.4: resolution: {integrity: sha512-43zkIKubNbnrULWlHdN5h1g3SEKXOEzoAlRsHOTFpnlDu8JlAOZSMJBLULusuXRequboiwJcj5vtYXKB3k7+2g==} engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} @@ -5305,7 +5336,6 @@ packages: /methods@1.1.2: resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} engines: {node: '>= 0.6'} - dev: false /micromatch@3.1.10: resolution: {integrity: sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==} @@ -6010,6 +6040,12 @@ packages: minipass: 5.0.0 dev: true + /path-to-regexp@1.8.0: + resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==} + dependencies: + isarray: 0.0.1 + dev: true + /path-to-regexp@6.2.1: resolution: {integrity: sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw==} dev: false @@ -7881,7 +7917,6 @@ packages: optional: true utf-8-validate: optional: true - dev: false /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} From 84495e9de35e1ecf6b85a09512ca610d104c40ce Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 18 Sep 2023 18:52:10 +0100 Subject: [PATCH 063/232] rtm: exploring mock server setup --- packages/rtm-server/socket4.js | 12 ++++++++++++ packages/rtm-server/src/mock/lightning/api.ts | 13 ++++++++----- packages/rtm-server/src/mock/lightning/server.ts | 10 +++++++++- packages/rtm-server/src/mock/socket-server.ts | 16 ++++++++++------ packages/rtm-server/test/mock/lightning.test.ts | 11 +++++++---- 5 files changed, 46 insertions(+), 16 deletions(-) diff --git a/packages/rtm-server/socket4.js b/packages/rtm-server/socket4.js index 3b46a5123..40c6f8ea5 100644 --- a/packages/rtm-server/socket4.js +++ b/packages/rtm-server/socket4.js @@ -1,12 +1,24 @@ // This attempt builds my own websocket into koa, only available at one path // Then I can use the connection handler myself to plug in my phoenix mock + import Koa from 'koa'; +import Router from '@koa/router'; import Socket, { WebSocketServer } from 'ws'; const app = new Koa(); const server = app.listen(3333); +app.use((ctx) => { + ctx.res; + ponse.status = 200; + return; +}); + +const r = new Router(); +r.all('/', () => {}); +app.use(r.routes()); + const wss = new WebSocketServer({ server, path: '/jam', diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 346424385..6351fe0cb 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -31,9 +31,12 @@ export interface AttemptCompleteBody extends RTMBody { // what if I do the server side mock in koa, can I use the pheonix client to connect? export const createNewAPI = (state: ServerState, path: string, server) => { // set up a websocket server to listen to connections + // console.log('path', path); const wss = new WebSocketServer({ server, - path, + + // If we set a path here, the websocket never seems to catch the connection + // path, }); // pass that through to the phoenix mock @@ -69,10 +72,10 @@ export const createNewAPI = (state: ServerState, path: string, server) => { // return handleEvent; - return (ctx) => { - // what does this actually do? - console.log(' >> ', ctx); - }; + // return (ctx) => { + // // what does this actually do? + // console.log(' >> ', ctx); + // }; }; // Note that this API is hosted at api/1 diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 677f2e2ad..d62810e3f 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'node:events'; import Koa from 'koa'; +import URL from 'node:url'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import websockify from 'koa-websocket'; @@ -51,6 +52,13 @@ const createLightningServer = (options: LightningOptions = {}) => { const app = new Koa(); app.use(bodyParser()); + app.use((ctx) => { + // ths is needed to get the websocket to recognise the reques tpath + ctx.req.path = URL.parse(req.url).pathname; + + console.log(' >> ', ctx.request.path); + }); + // this router but doesn't really seem to work // app.use(route.all('/websocket', createNewAPI(state, options.port || 8888))); @@ -87,7 +95,7 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(klogger); // Mock API endpoints - app.use(createAPI(state)); + // app.use(createAPI(state)); app.use(createDevAPI(app as any, state, logger)); app.destroy = () => { diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/socket-server.ts index b8acb5239..c5278c6cf 100644 --- a/packages/rtm-server/src/mock/socket-server.ts +++ b/packages/rtm-server/src/mock/socket-server.ts @@ -20,11 +20,11 @@ function createServer({ port = 8080, server } = {}) { // console.log('ws listening on', port); const channels: Record> = {}; - const wsServer = server; - // server || - // new WebSocketServer({ - // port, - // }); + const wsServer = + server || + new WebSocketServer({ + port, + }); const events = { // testing (TODO shouldn't this be in a specific channel?) @@ -55,8 +55,12 @@ function createServer({ port = 8080, server } = {}) { }, }; - wsServer.on('connection', function (ws: WS) { + wsServer.on('connection', function (ws: WS, req) { console.log(' >> connect'); + // console.log(ws); + // console.log(req); + console.log(req.url); + // console.log(req.path); ws.on('message', function (data: string) { console.log(' >> message'); const evt = JSON.parse(data) as PhoenixEvent; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index bed3e3bca..fd1e46034 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -40,9 +40,12 @@ const post = (path: string, data: any) => const attempt1 = attempts()['attempt-1']; -test.serial.skip('provide a phoenix websocket at /websocket', (t) => { +test.serial.only('provide a phoenix websocket at /websocket', (t) => { return new Promise(async (done) => { - const socket = new Socket(`ws://localhost:7777/websocket`); + // await fetch('http://localhost:7777/websocket'); + // in the mock the URL doesn't actually matter + // hang on, I think the pheonix socket my add /socket to the url or something? + const socket = new Socket(`ws://localhost:7777/jam`); socket.connect(); await sleep(); // TODO untidy @@ -51,7 +54,7 @@ test.serial.skip('provide a phoenix websocket at /websocket', (t) => { }); }); -test.serial('respond to connection join requests', (t) => { +test.serial.only('respond to connection join requests', (t) => { return new Promise(async (done) => { const socket = new Socket(`ws://localhost:7777`); //const socket = new Socket(`ws://localhost:7777/websocket`); // TODO this breaks??? @@ -69,7 +72,7 @@ test.serial('respond to connection join requests', (t) => { // Thinking a bit about messaging flow // a) it's not working (no connect, no join) // b) the way this is written is awful -test.serial('get a reply to a ping event', (t) => { +test.serial.only('get a reply to a ping event', (t) => { return new Promise(async (done) => { let didGetReply = false; const socket = new Socket(`ws://localhost:7777`); From 273e66a65a7137c9231685ddf57905e671566fa7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 18 Sep 2023 19:05:40 +0100 Subject: [PATCH 064/232] rtm: possibly fixed pathing --- packages/rtm-server/src/mock/lightning/api.ts | 5 +-- .../rtm-server/src/mock/lightning/server.ts | 43 +------------------ .../rtm-server/test/mock/lightning.test.ts | 16 +++---- 3 files changed, 10 insertions(+), 54 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 6351fe0cb..399aae411 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -35,13 +35,12 @@ export const createNewAPI = (state: ServerState, path: string, server) => { const wss = new WebSocketServer({ server, - // If we set a path here, the websocket never seems to catch the connection - // path, + // Note: phoenix websocket will connect to /websocket + path: path ? `${path}/websocket` : undefined, }); // pass that through to the phoenix mock createPheonixMockSocketServer({ server: wss }); - console.log('mock created'); // then do something clever to map events // server.on({ diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index d62810e3f..50c150536 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -44,52 +44,13 @@ const createLightningServer = (options: LightningOptions = {}) => { events: new EventEmitter(), } as ServerState; - // Server setup - // this seems to setup websockets to work at any path - // Maybe that's fine for the mock? Kind wierd - // mind you I still don't get a connect event - // const app = websockify(new Koa()); const app = new Koa(); app.use(bodyParser()); - app.use((ctx) => { - // ths is needed to get the websocket to recognise the reques tpath - ctx.req.path = URL.parse(req.url).pathname; - - console.log(' >> ', ctx.request.path); - }); - - // this router but doesn't really seem to work - // app.use(route.all('/websocket', createNewAPI(state, options.port || 8888))); - - // Using routes (seems to not work - at least the api doesn't get) - // app.ws.use( - // route.all('/websocket', (ctx) => createNewAPI(state, ctx.websocket)) - // ); - - // this probaably isn;t right because it'll create a new API object - /// for each request - const server = app.listen(options.port || 8888); - createNewAPI(state, '/websocket', server); - - // I really don't think this should be hard - // we connect to a socket sitting at /socket - // then we sub to events on both sides - - // app.ws.use( - // route.all('/websocket', function (ctx) { - // console.log(' >> WEBSOCKET '); - // // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. - // // the websocket is added to the context on `ctx.websocket`. - // ctx.websocket.send('Hello World'); - // ctx.websocket.on('message', function (message) { - // // do something with the message from client - // console.log(message); - // }); - // }) - // ); + // Setup the websocket API + createNewAPI(state, '/api', server); const klogger = koaLogger((str) => logger.debug(str)); app.use(klogger); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index fd1e46034..595c833dc 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -40,12 +40,9 @@ const post = (path: string, data: any) => const attempt1 = attempts()['attempt-1']; -test.serial.only('provide a phoenix websocket at /websocket', (t) => { +test.serial('provide a phoenix websocket at /websocket', (t) => { return new Promise(async (done) => { - // await fetch('http://localhost:7777/websocket'); - // in the mock the URL doesn't actually matter - // hang on, I think the pheonix socket my add /socket to the url or something? - const socket = new Socket(`ws://localhost:7777/jam`); + const socket = new Socket(`ws://localhost:7777/api`); socket.connect(); await sleep(); // TODO untidy @@ -54,10 +51,9 @@ test.serial.only('provide a phoenix websocket at /websocket', (t) => { }); }); -test.serial.only('respond to connection join requests', (t) => { +test.serial('respond to connection join requests', (t) => { return new Promise(async (done) => { - const socket = new Socket(`ws://localhost:7777`); - //const socket = new Socket(`ws://localhost:7777/websocket`); // TODO this breaks??? + const socket = new Socket(`ws://localhost:7777/api`); socket.connect(); const channel = socket.channel('x', {}); @@ -72,10 +68,10 @@ test.serial.only('respond to connection join requests', (t) => { // Thinking a bit about messaging flow // a) it's not working (no connect, no join) // b) the way this is written is awful -test.serial.only('get a reply to a ping event', (t) => { +test.serial('get a reply to a ping event', (t) => { return new Promise(async (done) => { let didGetReply = false; - const socket = new Socket(`ws://localhost:7777`); + const socket = new Socket(`ws://localhost:7777/api`); socket.connect(); // join the worker pool From a4297cde14f7283c8631594d51c25d1756dbf2e9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 19 Sep 2023 12:29:44 +0100 Subject: [PATCH 065/232] rtm: build up attempts/queue API --- packages/rtm-server/src/events.ts | 13 +- packages/rtm-server/src/mock/lightning/api.ts | 50 ++++- .../rtm-server/src/mock/lightning/server.ts | 3 +- packages/rtm-server/src/mock/socket-server.ts | 18 +- .../rtm-server/test/mock/lightning.test.ts | 207 ++++++++++-------- 5 files changed, 171 insertions(+), 120 deletions(-) diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 1b108c55d..3e016919f 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -1,19 +1,18 @@ // track socket event names as constants to keep refactoring easier -const CLAIM = 'attempt:claim'; +export const CLAIM = 'attempt:claim'; // TODO does each worker connect to its own channel, ensuring a private claim steeam? // or is there a shared Workers channel // claim reply needs to include the id of the server and the attempt -const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: 'a1' } - +export const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: 'a1' } // All attempt events are in a dedicated channel for that event -const ATTEMPT_START = 'attempt:start' // attemptId, timestamp -const ATTEMPT_COMPLETE = 'attempt:complete' // attemptId, timestamp, result, stats -const ATTEMPT_LOG = 'attempt:complete' // level, namespace (job,runtime,adaptor), message, time +export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp +export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats +export const ATTEMPT_LOG = 'attempt:complete'; // level, namespace (job,runtime,adaptor), message, time // this should not happen - this is "could not execute" rather than "complete with errors" -const ATTEMPT_ERROR = 'attempt:error' \ No newline at end of file +export const ATTEMPT_ERROR = 'attempt:error'; diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 399aae411..49169345f 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -3,7 +3,7 @@ import Socket, { WebSocketServer } from 'ws'; import { unimplemented, createListNextJob, - createFetchNextJob, + createClaim, createGetCredential, createLog, createComplete, @@ -13,6 +13,7 @@ import type { ServerState } from './server'; import { API_PREFIX } from './server'; import createPheonixMockSocketServer from '../socket-server'; +import { CLAIM } from '../../events'; interface RTMBody { rtm_id: string; @@ -24,29 +25,58 @@ export interface AttemptCompleteBody extends RTMBody { state: any; // JSON state object (undefined? null?) } +// pull claim will try and pull a claim off the queue, +// and reply with the response +// the reply ensures that only the calling worker will get the attempt +const pullClaim = (state, ws, evt) => { + const { ref, topic } = evt; + const { queue } = state; + let count = 1; + + const payload = { + status: 'ok', + response: [], + }; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + const next = queue.shift(); + payload.response.push(next.id); + count -= 1; + } + + ws.send( + JSON.stringify({ + event: `chan_reply_${ref}`, + ref, + topic, + payload, + }) + ); +}; + // this new API is websocket based // Events map to handlers // can I even implement this in JS? Not with pheonix anyway. hmm. // dead at the first hurdle really. // what if I do the server side mock in koa, can I use the pheonix client to connect? -export const createNewAPI = (state: ServerState, path: string, server) => { +export const createNewAPI = (state: ServerState, path: string, httpServer) => { // set up a websocket server to listen to connections // console.log('path', path); - const wss = new WebSocketServer({ - server, + const server = new WebSocketServer({ + server: httpServer, // Note: phoenix websocket will connect to /websocket path: path ? `${path}/websocket` : undefined, }); // pass that through to the phoenix mock - createPheonixMockSocketServer({ server: wss }); + const wss = createPheonixMockSocketServer({ server }); - // then do something clever to map events - // server.on({ - // 'attempt:claim': noop, - // 'attempt:start': noop, - // }) + wss.registerEvents('workers', { + [CLAIM]: (ws, event) => pullClaim(state, ws, event), + }); const noop = () => {}; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 50c150536..596f042e0 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -56,11 +56,12 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(klogger); // Mock API endpoints + // TODO should we keep the REST interface for local debug? + // Maybe for the read-only stuff (like get all attempts) // app.use(createAPI(state)); app.use(createDevAPI(app as any, state, logger)); app.destroy = () => { - console.log('close'); server.close(); }; diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/socket-server.ts index c5278c6cf..f4b771fdc 100644 --- a/packages/rtm-server/src/mock/socket-server.ts +++ b/packages/rtm-server/src/mock/socket-server.ts @@ -29,7 +29,6 @@ function createServer({ port = 8080, server } = {}) { const events = { // testing (TODO shouldn't this be in a specific channel?) ping: (ws, { topic, ref }) => { - console.log(' >> ping'); ws.send( JSON.stringify({ topic, @@ -41,7 +40,6 @@ function createServer({ port = 8080, server } = {}) { }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket phx_join: (ws, { topic, ref }) => { - console.log('-- join --'); ws.send( JSON.stringify({ // here is the magic reply event @@ -56,13 +54,7 @@ function createServer({ port = 8080, server } = {}) { }; wsServer.on('connection', function (ws: WS, req) { - console.log(' >> connect'); - // console.log(ws); - // console.log(req); - console.log(req.url); - // console.log(req.path); ws.on('message', function (data: string) { - console.log(' >> message'); const evt = JSON.parse(data) as PhoenixEvent; if (evt.topic) { // phx sends this info in each message @@ -75,7 +67,7 @@ function createServer({ port = 8080, server } = {}) { // handle custom/user events if (channels[topic]) { channels[topic].forEach((fn) => { - fn(event, payload); + fn(ws, { event, topic, payload, ref }); }); } } @@ -112,6 +104,14 @@ function createServer({ port = 8080, server } = {}) { }); }; + // TODO how do we unsubscribe? + wsServer.registerEvents = (topic: Topic, events) => { + for (const evt in events) { + console.log(evt); + wsServer.listenToChannel(topic, events[evt]); + } + }; + return wsServer; } diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 595c833dc..66844d750 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -4,7 +4,10 @@ import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; import { createMockLogger } from '@openfn/logger'; import phx from 'phoenix-channels'; -const { Socket } = phx; + +import { CLAIM } from '../../src/events'; + +const endpoint = 'ws://localhost:7777/api'; const baseUrl = `http://localhost:7777${API_PREFIX}`; @@ -14,10 +17,19 @@ const sleep = (duration = 10) => }); let server; - -test.before(() => { - server = createLightningServer({ port: 7777 }); -}); +let client; + +// Set up a lightning server and a phoenix socket client before each test +test.before( + () => + new Promise((done) => { + server = createLightningServer({ port: 7777 }); + + client = new phx.Socket(endpoint); + client.connect(); + client.onOpen(done); + }) +); test.afterEach(() => { server.reset(); @@ -27,6 +39,14 @@ test.after(() => { server.destroy(); }); +const join = (channelName: string): Promise => + new Promise((done) => { + const channel = client.channel(channelName, {}); + channel.join().receive('ok', () => { + done(channel); + }); + }); + const get = (path: string) => fetch(`${baseUrl}/${path}`); const post = (path: string, data: any) => fetch(`${baseUrl}/${path}`, { @@ -40,23 +60,14 @@ const post = (path: string, data: any) => const attempt1 = attempts()['attempt-1']; -test.serial('provide a phoenix websocket at /websocket', (t) => { - return new Promise(async (done) => { - const socket = new Socket(`ws://localhost:7777/api`); - - socket.connect(); - await sleep(); // TODO untidy - t.is(socket.connectionState(), 'open'); - done(); - }); +test.serial('provide a phoenix websocket at /api', (t) => { + // client should be connected before this test runs + t.is(client.connectionState(), 'open'); }); test.serial('respond to connection join requests', (t) => { return new Promise(async (done) => { - const socket = new Socket(`ws://localhost:7777/api`); - - socket.connect(); - const channel = socket.channel('x', {}); + const channel = client.channel('x', {}); channel.join().receive('ok', (resp) => { t.is(resp, 'ok'); @@ -65,37 +76,100 @@ test.serial('respond to connection join requests', (t) => { }); }); -// Thinking a bit about messaging flow -// a) it's not working (no connect, no join) -// b) the way this is written is awful +// TODO: only allow authorised workers to join workers +// TODO: only allow authorised attemtps to join an attempt channel + test.serial('get a reply to a ping event', (t) => { return new Promise(async (done) => { - let didGetReply = false; - const socket = new Socket(`ws://localhost:7777/api`); + const channel = await join('test'); - socket.connect(); - // join the worker pool - const channel = socket.channel('workers', {}); - channel.join().receive('ok', () => { - // should get a response - channel.on('pong', (payload) => { - console.log('[ping] reply', payload); - didGetReply = true; + channel.on('pong', (payload) => { + t.pass('message received'); + done(); + }); - t.true(didGetReply); + channel.push('ping'); + }); +}); + +test.serial.only( + 'claim attempt: reply for zero items if queue is empty', + (t) => + new Promise(async (done) => { + t.is(server.getQueueLength(), 0); + + const channel = await join('workers'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + t.assert(Array.isArray(response)); + t.is(response.length, 0); + + t.is(server.getQueueLength(), 0); done(); }); + }) +); - // TODO explicit test that the backing socket got this event? - channel.push('ping'); - }); - }); -}); +test.serial.only( + "claim attempt: reply with an attempt id if there's an attempt in the queue", + (t) => + new Promise(async (done) => { + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 1); + + // This uses a shared channel at all workers sit in + // They all yell from time to time to ask for work + // Lightning responds with an attempt id and server id (target) + // What if: + // a) each worker has its own channel, so claims are handed out privately + // b) we use the 'ok' status to return work in the response + // this b pattern is much nicer + const channel = await join('workers'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + t.truthy(response); + t.is(response.length, 1); + t.is(response[0], 'attempt-1'); + + // ensure the server state has changed + t.is(server.getQueueLength(), 0); + done(); + }); + }) +); -// respond to a claim request with an id -// uh does this stuff make any sense in the socket model? +// TODO is it even worth doing this? Easier for a socket to pull one at a time? +// It would also ensure better distribution if 10 workers ask at the same time, they'll get +// one each then come back for more +test.serial.skip( + 'claim attempt: reply with multiple attempt ids', + (t) => + new Promise(async (done) => { + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 3); + + const channel = await join('workers'); + + // // response is an array of attempt ids + // channel.push(CLAIM, { count: 3 }).receive('ok', (response) => { + // t.truthy(response); + // t.is(response.length, 1); + // t.is(response[0], 'attempt-1'); + + // // ensure the server state has changed + // t.is(server.getQueueLength(), 0); + // done(); + // }); + }) +); -// create a channel for an attempt +// TODO get execution plan +// TODO get credentials +// TODO get state test.serial.skip( 'GET /credential - return 404 if no credential found', @@ -117,24 +191,6 @@ test.serial.skip('GET /credential - return a credential', async (t) => { t.is(job.password, 'cash'); }); -test.serial.skip( - 'POST /attempts/next - return 204 and no body for an empty queue', - async (t) => { - t.is(server.getQueueLength(), 0); - const res = await post('attempts/next', { rtm_id: 'rtm' }); - t.is(res.status, 204); - t.false(res.bodyUsed); - } -); - -test.serial.skip( - 'POST /attempts/next - return 400 if no id provided', - async (t) => { - const res = await post('attempts/next', {}); - t.is(res.status, 400); - } -); - test.serial.skip( 'GET /attempts/next - return 200 with a workflow', async (t) => { @@ -179,41 +235,6 @@ test.serial.skip( } ); -test.serial.skip( - 'GET /attempts/next - return 200 with 2 workflows', - async (t) => { - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 3); - - const res = await post('attempts/next?count=2', { rtm_id: 'rtm' }); - t.is(res.status, 200); - - const result = await res.json(); - t.truthy(result); - t.true(Array.isArray(result)); - t.is(result.length, 2); - - t.is(server.getQueueLength(), 1); - } -); - -test.serial.skip( - 'POST /attempts/next - clear the queue after a request', - async (t) => { - server.enqueueAttempt(attempt1); - const res1 = await post('attempts/next', { rtm_id: 'rtm' }); - t.is(res1.status, 200); - - const result1 = await res1.json(); - t.is(result1.length, 1); - const res2 = await post('attempts/next', { rtm_id: 'rtm' }); - t.is(res2.status, 204); - t.falsy(res2.bodyUsed); - } -); - test.serial.skip('POST /attempts/log - should return 200', async (t) => { server.enqueueAttempt(attempt1); const { status } = await post('attempts/log/attempt-1', { From 7bfb0158a9bd747e8ad91e55a0520a01c75bdb2a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 19 Sep 2023 15:11:24 +0100 Subject: [PATCH 066/232] rtm: feed attempt data through the socket --- packages/rtm-server/src/events.ts | 3 + .../rtm-server/src/mock/lightning/api-dev.ts | 13 +- packages/rtm-server/src/mock/lightning/api.ts | 128 ++++++++++-------- .../src/mock/lightning/middleware.ts | 61 ++++----- .../rtm-server/src/mock/lightning/server.ts | 25 ++-- .../src/mock/{ => lightning}/socket-server.ts | 20 ++- .../rtm-server/src/mock/lightning/util.ts | 4 + .../rtm-server/test/mock/lightning.test.ts | 53 ++++++-- .../test/mock/socket-server.test.ts | 6 +- 9 files changed, 193 insertions(+), 120 deletions(-) rename packages/rtm-server/src/mock/{ => lightning}/socket-server.ts (82%) create mode 100644 packages/rtm-server/src/mock/lightning/util.ts diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 3e016919f..290546a9b 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -10,6 +10,9 @@ export const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: // All attempt events are in a dedicated channel for that event +// or attempt_get ? I think there are several getters so maybe this makes sense +export const GET_ATTEMPT = 'fetch:attempt'; + export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats export const ATTEMPT_LOG = 'attempt:complete'; // level, namespace (job,runtime,adaptor), message, time diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index 692c13cf3..6d1860e3b 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -23,7 +23,7 @@ export type DevApp = Koa & { once(event: LightningEvents, fn: (evt: any) => void): void; }; -const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger) => { +const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { // Dev APIs for unit testing app.addCredential = (id: string, cred: Credential) => { logger.info(`Add credential ${id}`); @@ -65,6 +65,13 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger) => { app.getResult = (attemptId: string) => state.results[attemptId]?.state; + // TODO maybe onAttemptClaimed? or claim attempt? + app.startAttempt = (attemptId: string) => api.startAttempt(attemptId); + + app.registerAttempt = (attempt: any) => { + state.attempts[attempt.id] = attempt; + }; + // TODO these are overriding koa's event handler - should I be doing something different? // @ts-ignore @@ -98,7 +105,7 @@ const setupRestAPI = (app: DevApp, _state: ServerState, logger: Logger) => { return router.routes(); }; -export default (app: DevApp, state: ServerState, logger: Logger) => { - setupDevAPI(app, state, logger); +export default (app: DevApp, state: ServerState, logger: Logger, api) => { + setupDevAPI(app, state, logger, api); return setupRestAPI(app, state, logger); }; diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 49169345f..762d78b79 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -11,9 +11,10 @@ import { import type { ServerState } from './server'; import { API_PREFIX } from './server'; +import { extractAttemptId } from './util'; -import createPheonixMockSocketServer from '../socket-server'; -import { CLAIM } from '../../events'; +import createPheonixMockSocketServer from './socket-server'; +import { CLAIM, GET_ATTEMPT } from '../../events'; interface RTMBody { rtm_id: string; @@ -25,37 +26,6 @@ export interface AttemptCompleteBody extends RTMBody { state: any; // JSON state object (undefined? null?) } -// pull claim will try and pull a claim off the queue, -// and reply with the response -// the reply ensures that only the calling worker will get the attempt -const pullClaim = (state, ws, evt) => { - const { ref, topic } = evt; - const { queue } = state; - let count = 1; - - const payload = { - status: 'ok', - response: [], - }; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - const next = queue.shift(); - payload.response.push(next.id); - count -= 1; - } - - ws.send( - JSON.stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload, - }) - ); -}; - // this new API is websocket based // Events map to handlers // can I even implement this in JS? Not with pheonix anyway. hmm. @@ -72,39 +42,81 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { }); // pass that through to the phoenix mock - const wss = createPheonixMockSocketServer({ server }); + const wss = createPheonixMockSocketServer({ server, state }); + + // pull claim will try and pull a claim off the queue, + // and reply with the response + // the reply ensures that only the calling worker will get the attempt + const pullClaim = (state, ws, evt) => { + const { ref, topic } = evt; + const { queue } = state; + let count = 1; + + const payload = { + status: 'ok', + response: [], + }; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + const next = queue.shift(); + payload.response.push(next.id); + count -= 1; + + startAttempt(next.id); + } + + ws.send( + JSON.stringify({ + event: `chan_reply_${ref}`, + ref, + topic, + payload, + }) + ); + }; + + const getAttempt = (state, ws, evt) => { + const { ref, topic } = evt; + const attemptId = extractAttemptId(topic); + const attempt = state.attempts[attemptId]; + + ws.send( + JSON.stringify({ + event: `chan_reply_${ref}`, + ref, + topic, + payload: { + status: 'ok', + response: attempt, + }, + }) + ); + }; wss.registerEvents('workers', { [CLAIM]: (ws, event) => pullClaim(state, ws, event), - }); - - const noop = () => {}; - // This may actually get split into a server bit and a an attempts bit, reflecting the different channels - const events = { - hello: noop, + // is this part of the general workers pool, or part of the attempt? + // probably part of the attempt because we can control permissions + }); - 'attempt:claim': noop, - 'attempt:start': noop, - 'attempt:complete': noop, - 'attempt:get_credential': noop, - 'attempt:credential': noop, - 'attempt:get_dataclip': noop, - 'attempt:dataclip': noop, + const startAttempt = (attemptId) => { + // mark the attempt as started on the server + // TODO right now this duplicates logic in the dev API + state.pending[attemptId] = { + status: 'started', + }; - 'run:start': noop, - 'run:end': noop, - 'run:log': noop, + wss.registerEvents(`attempt:${attemptId}`, { + [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), + }); }; - // const handleEvent = (name) => {}; - - // return handleEvent; - - // return (ctx) => { - // // what does this actually do? - // console.log(' >> ', ctx); - // }; + return { + startAttempt, + }; }; // Note that this API is hosted at api/1 diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 1f4e2a272..160b52cf1 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -19,39 +19,38 @@ export const unimplemented = (ctx: Koa.Context) => { ctx.status = 501; }; -export const createFetchNextJob = - (state: ServerState) => (ctx: Koa.Context) => { - const { queue } = state; - const { body } = ctx.request; - if (!body || !body.rtm_id) { - ctx.status = 400; - return; - } - const countRaw = ctx.request.query.count as unknown; - let count = 1; - if (countRaw) { - if (!isNaN(countRaw)) { - count = countRaw as number; - } else { - console.error('Failed to parse parameter countRaw'); - } - } - const payload = []; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - payload.push(queue.shift()); - count -= 1; - } - if (payload.length > 0) { - ctx.body = JSON.stringify(payload); - ctx.status = 200; +export const createClaim = (state: ServerState) => (ctx: Koa.Context) => { + const { queue } = state; + const { body } = ctx.request; + if (!body || !body.rtm_id) { + ctx.status = 400; + return; + } + const countRaw = ctx.request.query.count as unknown; + let count = 1; + if (countRaw) { + if (!isNaN(countRaw)) { + count = countRaw as number; } else { - ctx.body = undefined; - ctx.status = 204; + console.error('Failed to parse parameter countRaw'); } - }; + } + const payload = []; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + payload.push(queue.shift()); + count -= 1; + } + if (payload.length > 0) { + ctx.body = JSON.stringify(payload); + ctx.status = 200; + } else { + ctx.body = undefined; + ctx.status = 204; + } +}; export const createListNextJob = (state: ServerState) => (ctx: Koa.Context) => { ctx.body = state.queue.map(({ id }) => id); diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 596f042e0..cc7a93ab9 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -7,7 +7,7 @@ import websockify from 'koa-websocket'; import route from 'koa-route'; import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; -import createServer from '../socket-server'; +import createServer from './socket-server'; import createAPI, { createNewAPI } from './api'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; @@ -28,18 +28,27 @@ export type LightningOptions = { port?: string | number; }; +export type AttemptId = string; + // a mock lightning server const createLightningServer = (options: LightningOptions = {}) => { const logger = options.logger || createMockLogger(); - // App state websockify = require('koa-websocket'); const state = { + // list of credentials by id credentials: {}, - attempts: [], + // list of events by id + attempts: {}, + + // list of dataclips by id + dataclips: {}, + + // attempts which have been started + // probaby need to track status and maybe the rtm id? + // TODO maybe Active is a better word? + pending: {}, - // TODO for now, the queue will hold the actual Attempt data directly - // I think later we want it to just hold an id? - queue: [] as Attempt[], + queue: [] as AttemptId[], results: {}, events: new EventEmitter(), } as ServerState; @@ -50,7 +59,7 @@ const createLightningServer = (options: LightningOptions = {}) => { const server = app.listen(options.port || 8888); // Setup the websocket API - createNewAPI(state, '/api', server); + const api = createNewAPI(state, '/api', server); const klogger = koaLogger((str) => logger.debug(str)); app.use(klogger); @@ -59,7 +68,7 @@ const createLightningServer = (options: LightningOptions = {}) => { // TODO should we keep the REST interface for local debug? // Maybe for the read-only stuff (like get all attempts) // app.use(createAPI(state)); - app.use(createDevAPI(app as any, state, logger)); + app.use(createDevAPI(app as any, state, logger, api)); app.destroy = () => { server.close(); diff --git a/packages/rtm-server/src/mock/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts similarity index 82% rename from packages/rtm-server/src/mock/socket-server.ts rename to packages/rtm-server/src/mock/lightning/socket-server.ts index f4b771fdc..2fd06c74e 100644 --- a/packages/rtm-server/src/mock/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -1,4 +1,6 @@ import { WebSocketServer } from 'ws'; +import { ATTEMPT_PREFIX, extractAttemptId } from './util'; + // mock pheonix websocket server // - route messages to rooms @@ -16,7 +18,7 @@ type PhoenixEvent = { type EventHandler = (event: string, payload: any) => void; -function createServer({ port = 8080, server } = {}) { +function createServer({ port = 8080, server, state } = {}) { // console.log('ws listening on', port); const channels: Record> = {}; @@ -39,14 +41,25 @@ function createServer({ port = 8080, server } = {}) { ); }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket - phx_join: (ws, { topic, ref }) => { + phx_join: (ws, { event, topic, ref }) => { + let status = 'ok'; + let response = 'ok'; + + // TODO is this logic in the right place? + if (topic.startsWith(ATTEMPT_PREFIX)) { + const attemptId = extractAttemptId(topic); + if (!state.pending[attemptId]) { + status = 'error'; + response = 'invalid_attempt'; + } + } ws.send( JSON.stringify({ // here is the magic reply event // see channel.replyEventName event: `chan_reply_${ref}`, topic, - payload: { status: 'ok', response: 'ok' }, + payload: { status, response }, ref, }) ); @@ -107,7 +120,6 @@ function createServer({ port = 8080, server } = {}) { // TODO how do we unsubscribe? wsServer.registerEvents = (topic: Topic, events) => { for (const evt in events) { - console.log(evt); wsServer.listenToChannel(topic, events[evt]); } }; diff --git a/packages/rtm-server/src/mock/lightning/util.ts b/packages/rtm-server/src/mock/lightning/util.ts new file mode 100644 index 000000000..6fd5685e1 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/util.ts @@ -0,0 +1,4 @@ +export const ATTEMPT_PREFIX = 'attempt:'; + +export const extractAttemptId = (topic: string) => + topic.substr(ATTEMPT_PREFIX.length); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 66844d750..da7e13c98 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -5,7 +5,7 @@ import { createMockLogger } from '@openfn/logger'; import phx from 'phoenix-channels'; -import { CLAIM } from '../../src/events'; +import { CLAIM, GET_ATTEMPT } from '../../src/events'; const endpoint = 'ws://localhost:7777/api'; @@ -25,7 +25,7 @@ test.before( new Promise((done) => { server = createLightningServer({ port: 7777 }); - client = new phx.Socket(endpoint); + client = new phx.Socket(endpoint, { timeout: 50 }); client.connect(); client.onOpen(done); }) @@ -40,11 +40,16 @@ test.after(() => { }); const join = (channelName: string): Promise => - new Promise((done) => { + new Promise((done, reject) => { const channel = client.channel(channelName, {}); - channel.join().receive('ok', () => { - done(channel); - }); + channel + .join() + .receive('ok', () => { + done(channel); + }) + .receive('error', (err) => { + reject(new Error(err)); + }); }); const get = (path: string) => fetch(`${baseUrl}/${path}`); @@ -66,11 +71,11 @@ test.serial('provide a phoenix websocket at /api', (t) => { }); test.serial('respond to connection join requests', (t) => { - return new Promise(async (done) => { + return new Promise(async (done, reject) => { const channel = client.channel('x', {}); - channel.join().receive('ok', (resp) => { - t.is(resp, 'ok'); + channel.join().receive('ok', (res) => { + t.is(res, 'ok'); done(); }); }); @@ -92,7 +97,7 @@ test.serial('get a reply to a ping event', (t) => { }); }); -test.serial.only( +test.serial( 'claim attempt: reply for zero items if queue is empty', (t) => new Promise(async (done) => { @@ -111,7 +116,7 @@ test.serial.only( }) ); -test.serial.only( +test.serial( "claim attempt: reply with an attempt id if there's an attempt in the queue", (t) => new Promise(async (done) => { @@ -167,10 +172,34 @@ test.serial.skip( }) ); -// TODO get execution plan // TODO get credentials // TODO get state +test.serial('create a channel for an attempt', async (t) => { + server.startAttempt('wibble'); + await join('attempt:wibble'); + t.pass('connection ok'); +}); + +test.serial('reject channels for attempts that are not started', async (t) => { + await t.throwsAsync(() => join('attempt:wibble'), { + message: 'invalid_attempt', + }); +}); + +test.serial('get attempt data through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.registerAttempt(attempt1); + server.startAttempt(attempt1.id); + + const channel = await join(`attempt:${attempt1.id}`); + channel.push(GET_ATTEMPT).receive('ok', (p) => { + t.deepEqual(p, attempt1); + done(); + }); + }); +}); + test.serial.skip( 'GET /credential - return 404 if no credential found', async (t) => { diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index ebc06dbd7..a91fae39c 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -1,9 +1,7 @@ import test from 'ava'; - import phx from 'phoenix-channels'; -const { Socket } = phx; -import createServer from '../../src/mock/socket-server'; +import createServer from '../../src/mock/lightning/socket-server'; let socket; let server; @@ -16,7 +14,7 @@ const wait = (duration = 10) => test.beforeEach(() => { server = createServer(); - socket = new Socket('ws://localhost:8080'); + socket = new phx.Socket('ws://localhost:8080'); socket.connect(); }); From 52a0d0815999360443b73389c889ce25085e5366 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 19 Sep 2023 16:52:00 +0100 Subject: [PATCH 067/232] rtm: events for getting creds and data tests are failing though, somehow data doesn't get returned --- packages/rtm-server/src/events.ts | 2 + packages/rtm-server/src/mock/data.ts | 24 ----- .../rtm-server/src/mock/lightning/api-dev.ts | 5 + packages/rtm-server/src/mock/lightning/api.ts | 48 ++++++++- .../src/mock/lightning/socket-server.ts | 1 + packages/rtm-server/test/mock/data.ts | 31 ++++++ .../rtm-server/test/mock/lightning.test.ts | 98 +++++++------------ packages/rtm-server/tsconfig.json | 2 +- 8 files changed, 123 insertions(+), 88 deletions(-) delete mode 100644 packages/rtm-server/src/mock/data.ts create mode 100644 packages/rtm-server/test/mock/data.ts diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 290546a9b..030a28ca8 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -12,6 +12,8 @@ export const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: // or attempt_get ? I think there are several getters so maybe this makes sense export const GET_ATTEMPT = 'fetch:attempt'; +export const GET_CREDENTIAL = 'fetch:credential'; +export const GET_DATACLIP = 'fetch:dataclip'; export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats diff --git a/packages/rtm-server/src/mock/data.ts b/packages/rtm-server/src/mock/data.ts deleted file mode 100644 index d3254c325..000000000 --- a/packages/rtm-server/src/mock/data.ts +++ /dev/null @@ -1,24 +0,0 @@ -// TODO this file, if we really need it, should move into test - -export const credentials = () => ({ - a: { - user: 'bobby', - password: 'password1', - }, -}); - -export const attempts = () => ({ - 'attempt-1': { - id: 'attempt-1', - input: { - data: {}, - }, - plan: [ - { - adaptor: '@openfn/language-common@1.0.0', - expression: 'fn(a => a)', - credential: 'a', - }, - ], - }, -}); diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index 6d1860e3b..42fa3132f 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -30,6 +30,11 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { state.credentials[id] = cred; }; + app.addDataclip = (id: string, data: any) => { + logger.info(`Add dataclip ${id}`); + state.dataclips[id] = data; + }; + // Promise which returns when a workflow is complete app.waitForResult = (attemptId: string) => { return new Promise((resolve) => { diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 762d78b79..f939fb628 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -14,7 +14,7 @@ import { API_PREFIX } from './server'; import { extractAttemptId } from './util'; import createPheonixMockSocketServer from './socket-server'; -import { CLAIM, GET_ATTEMPT } from '../../events'; +import { CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP } from '../../events'; interface RTMBody { rtm_id: string; @@ -44,6 +44,11 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { // pass that through to the phoenix mock const wss = createPheonixMockSocketServer({ server, state }); + // TODO + // 1) Need to improve the abtraction of these, make messages easier to send + // 2) Also need to look at closures - I'd like a declarative central API + // the need to call startAttempt makes things a bit harder + // pull claim will try and pull a claim off the queue, // and reply with the response // the reply ensures that only the calling worker will get the attempt @@ -95,6 +100,41 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { ); }; + const getCredential = (state, ws, evt) => { + const { ref, topic, payload, event } = evt; + const response = state.credentials[payload.id]; + console.log(topic, event, response); + ws.send( + JSON.stringify({ + event: `chan_reply_${ref}`, + ref, + topic, + payload: { + status: 'ok', + response, + }, + }) + ); + }; + + const getDataclip = (state, ws, evt) => { + console.log(' getDataClip'); + const { ref, topic, payload, event } = evt; + const response = state.dataclips[payload.id]; + console.log(response); + ws.send( + JSON.stringify({ + event: `chan_reply_${ref}`, + ref, + topic, + payload: { + status: 'ok', + response, + }, + }) + ); + }; + wss.registerEvents('workers', { [CLAIM]: (ws, event) => pullClaim(state, ws, event), @@ -109,8 +149,14 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { status: 'started', }; + // TODO do all these need extra auth, or is auth granted + // implicitly by channel membership? + // Right now the socket gets access to all server state + // But this is just a mock - Lightning can impose more restrictions if it wishes wss.registerEvents(`attempt:${attemptId}`, { [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), + [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), + [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), }); }; diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 2fd06c74e..ffa79a69b 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -69,6 +69,7 @@ function createServer({ port = 8080, server, state } = {}) { wsServer.on('connection', function (ws: WS, req) { ws.on('message', function (data: string) { const evt = JSON.parse(data) as PhoenixEvent; + if (evt.topic) { // phx sends this info in each message const { topic, event, payload, ref } = evt; diff --git a/packages/rtm-server/test/mock/data.ts b/packages/rtm-server/test/mock/data.ts new file mode 100644 index 000000000..121107f31 --- /dev/null +++ b/packages/rtm-server/test/mock/data.ts @@ -0,0 +1,31 @@ +export const credentials = { + a: { + user: 'bobby', + password: 'password1', + }, +}; + +export const dataclips = { + d: { + count: 1, + }, +}; + +export const attempts = { + 'attempt-1': { + id: 'attempt-1', + // TODO how should this be structure? + input: { + data: 'd', + }, + triggers: [], + edges: [], + jobs: [ + { + adaptor: '@openfn/language-common@1.0.0', + body: 'fn(a => a)', + credential: 'a', + }, + ], + }, +}; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index da7e13c98..edac92a85 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,11 +1,16 @@ import test from 'ava'; -import { attempts } from '../../src/mock/data'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; import { createMockLogger } from '@openfn/logger'; import phx from 'phoenix-channels'; -import { CLAIM, GET_ATTEMPT } from '../../src/events'; +import { attempts, credentials, dataclips } from './data'; +import { + CLAIM, + GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, +} from '../../src/events'; const endpoint = 'ws://localhost:7777/api'; @@ -39,6 +44,8 @@ test.after(() => { server.destroy(); }); +const attempt1 = attempts['attempt-1']; + const join = (channelName: string): Promise => new Promise((done, reject) => { const channel = client.channel(channelName, {}); @@ -63,8 +70,6 @@ const post = (path: string, data: any) => }, }); -const attempt1 = attempts()['attempt-1']; - test.serial('provide a phoenix websocket at /api', (t) => { // client should be connected before this test runs t.is(client.connectionState(), 'open'); @@ -194,75 +199,44 @@ test.serial('get attempt data through the attempt channel', async (t) => { const channel = await join(`attempt:${attempt1.id}`); channel.push(GET_ATTEMPT).receive('ok', (p) => { + console.log('attempt', p); t.deepEqual(p, attempt1); done(); }); }); }); -test.serial.skip( - 'GET /credential - return 404 if no credential found', - async (t) => { - const res = await get('credential/x'); - t.is(res.status, 404); - } -); - -test.serial.skip('GET /credential - return a credential', async (t) => { - server.addCredential('a', { user: 'johnny', password: 'cash' }); - - const res = await get('credential/a'); - t.is(res.status, 200); - - const job = await res.json(); +// TODO can't work out why this is failing - there's just no data in the response +test.serial.skip('get credential through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.startAttempt(attempt1.id); + server.addCredential('a', credentials['a']); - t.is(job.user, 'johnny'); - t.is(job.password, 'cash'); + const channel = await join(`attempt:${attempt1.id}`); + channel.push(GET_CREDENTIAL, { id: 'a' }).receive('ok', (result) => { + t.deepEqual(result, credentials['a']); + done(); + }); + }); }); -test.serial.skip( - 'GET /attempts/next - return 200 with a workflow', - async (t) => { - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 1); - - const res = await post('attempts/next', { rtm_id: 'rtm' }); - const result = await res.json(); - t.is(res.status, 200); - - t.truthy(result); - t.true(Array.isArray(result)); - t.is(result.length, 1); - - // not interested in testing much against the attempt structure at this stage - const [attempt] = result; - t.is(attempt.id, 'attempt-1'); - t.true(Array.isArray(attempt.plan)); - - t.is(server.getQueueLength(), 0); - } -); - -test.serial.skip( - 'GET /attempts/next - return 200 with a workflow with an inline item', - async (t) => { - server.enqueueAttempt({ id: 'abc' }); - t.is(server.getQueueLength(), 1); - - const res = await post('attempts/next', { rtm_id: 'rtm' }); - t.is(res.status, 200); - - const result = await res.json(); - t.truthy(result); - t.true(Array.isArray(result)); - t.is(result.length, 1); +// TODO can't work out why this is failing - there's just no data in the response +test.serial.skip('get dataclip through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.startAttempt(attempt1.id); + server.addDataclip('d', dataclips['d']); - const [attempt] = result; - t.is(attempt.id, 'abc'); + const channel = await join(`attempt:${attempt1.id}`); + channel.push(GET_DATACLIP, { id: 'd' }).receive('ok', (result) => { + t.deepEqual(result, dataclips['d']); + done(); + }); + }); +}); - t.is(server.getQueueLength(), 0); - } -); +// TODO not going to bother testing attempt_ logs, they're a bit more passive in the process +// Lightning doesn't really care about them +// should we acknowledge them maybe though? test.serial.skip('POST /attempts/log - should return 200', async (t) => { server.enqueueAttempt(attempt1); diff --git a/packages/rtm-server/tsconfig.json b/packages/rtm-server/tsconfig.json index ba1452256..7d1bac97f 100644 --- a/packages/rtm-server/tsconfig.json +++ b/packages/rtm-server/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.common", - "include": ["src/**/*.ts"], + "include": ["src/**/*.ts", "test/mock/data.ts"], "compilerOptions": { "module": "ESNext" } From c0751f2ae9abe1f5f4c0205bbda4fb00c83830d3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 19 Sep 2023 17:10:31 +0100 Subject: [PATCH 068/232] rtm: remove some scratchpads --- packages/rtm-server/socket3.js | 66 --------------------------------- packages/rtm-server/socket4.js | 52 -------------------------- packages/rtm-server/sockets.js | 52 -------------------------- packages/rtm-server/sockets2.js | 8 ---- 4 files changed, 178 deletions(-) delete mode 100644 packages/rtm-server/socket3.js delete mode 100644 packages/rtm-server/socket4.js delete mode 100644 packages/rtm-server/sockets.js delete mode 100644 packages/rtm-server/sockets2.js diff --git a/packages/rtm-server/socket3.js b/packages/rtm-server/socket3.js deleted file mode 100644 index 2436eddeb..000000000 --- a/packages/rtm-server/socket3.js +++ /dev/null @@ -1,66 +0,0 @@ -// create a koa ws server -import Koa from 'koa'; -import route from 'koa-route'; -import websockify from 'koa-websocket'; -import Socket from 'ws'; - -// learnings: route is .all(path, fn) -// socket takes on 'message' - -const app = websockify(new Koa()); - -console.log(app.ws); - -app.ws.server.on('connection', () => { - console.log(' >> connect'); -}); - -// Regular middleware -// Note it's app.ws.use and not app.use -app.ws.use( - route.all('/jam', (ctx, next) => { - console.log('>> jam'); - - // I need this connection from the server, not the socket - // koa-ws hides that from me - - ctx.websocket.on('message', (m) => { - const x = m.toString(); - if (x === 'ping') { - console.log('received'); - ctx.websocket.send('pong'); - } - }); - - // return `next` to pass the context (ctx) on to the next ws middleware - return next(ctx); - }) -); - -// // Using routes -// app.ws.use(route.all('/test/:id', function (ctx) { -// // `ctx` is the regular koa context created from the `ws` onConnection `socket.upgradeReq` object. -// // the websocket is added to the context on `ctx.websocket`. -// ctx.websocket.send('Hello World'); -// ctx.websocket.on('message', function(message) { -// // do something with the message from client -// console.log(message); -// }); -// })); - -app.listen(3333); - -const s = new Socket('ws://localhost:3333/jam'); - -s.on('open', () => { - console.log('pinging...'); - s.send('ping'); -}); - -s.on('message', (m) => { - const message = m.toString(); - if (message === 'pong') { - console.log('pong!'); - process.exit(0); - } -}); diff --git a/packages/rtm-server/socket4.js b/packages/rtm-server/socket4.js deleted file mode 100644 index 40c6f8ea5..000000000 --- a/packages/rtm-server/socket4.js +++ /dev/null @@ -1,52 +0,0 @@ -// This attempt builds my own websocket into koa, only available at one path -// Then I can use the connection handler myself to plug in my phoenix mock - -import Koa from 'koa'; -import Router from '@koa/router'; -import Socket, { WebSocketServer } from 'ws'; - -const app = new Koa(); - -const server = app.listen(3333); - -app.use((ctx) => { - ctx.res; - ponse.status = 200; - return; -}); - -const r = new Router(); -r.all('/', () => {}); -app.use(r.routes()); - -const wss = new WebSocketServer({ - server, - path: '/jam', -}); - -wss.on('connection', (socket, req) => { - console.log('>> connection'); - socket.on('message', (m) => { - console.log(m); - const x = m.toString(); - if (x === 'ping') { - console.log('received'); - socket.send('pong'); - } - }); -}); - -const s = new Socket('ws://localhost:3333/jam'); - -s.on('open', () => { - console.log('pinging...'); - s.send('ping'); -}); - -s.on('message', (m) => { - const message = m.toString(); - if (message === 'pong') { - console.log('pong!'); - process.exit(0); - } -}); diff --git a/packages/rtm-server/sockets.js b/packages/rtm-server/sockets.js deleted file mode 100644 index 4d074996a..000000000 --- a/packages/rtm-server/sockets.js +++ /dev/null @@ -1,52 +0,0 @@ -import WebSocket, { WebSocketServer } from 'ws'; -import phx from 'phoenix-channels'; -import http from 'node:http'; - -const { Socket } = phx; - -/* - * web socket experiments - */ - -/* - Super simple ws implementation -*/ -const wsServer = new WebSocketServer({ - port: 8080, -}); - -wsServer.on('connection', function (ws) { - console.log('connection'); - - ws.on('message', function (data) { - console.log('server received: %s', data); - - // TMP - // process.exit(0); - }); -}); - -// const s = new WebSocket('ws://localhost:8080'); - -// // This bit is super important! Can't send ontil we've got the on open callback -// s.on('open', () => { -// console.log('sending...'); -// s.send('hello'); -// }); - -// This is a phoenix socket backing onto a normal websocket server -const s = new Socket('ws://localhost:8080'); -s.connect(); - -console.log('*'); -let channel = s.channel('room:lobby', {}); -channel.join().receive('ok', (resp) => { - console.log('Joined successfully', resp); - - channel.push('hello'); -}); -// .receive('error', (resp) => { -// console.log('Unable to join', resp); -// }); - -// channel.push('hello'); diff --git a/packages/rtm-server/sockets2.js b/packages/rtm-server/sockets2.js deleted file mode 100644 index 536f171fa..000000000 --- a/packages/rtm-server/sockets2.js +++ /dev/null @@ -1,8 +0,0 @@ -import WebSocket, { WebSocketServer } from 'ws'; - -const s = new WebSocket('ws://localhost:8080'); -s.on('open', () => { - console.log('sending...'); - s.send('hello'); - process.exit(0); -}); From 2fa884ed966493ecae5fd92599e292a00a9295c4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 20 Sep 2023 14:01:28 +0100 Subject: [PATCH 069/232] rtm: update queue name --- packages/rtm-server/src/mock/lightning/api.ts | 4 ++-- packages/rtm-server/src/server.ts | 6 ++++++ packages/rtm-server/test/mock/lightning.test.ts | 8 ++++---- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index f939fb628..31c85f690 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -103,7 +103,7 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { const getCredential = (state, ws, evt) => { const { ref, topic, payload, event } = evt; const response = state.credentials[payload.id]; - console.log(topic, event, response); + // console.log(topic, event, response); ws.send( JSON.stringify({ event: `chan_reply_${ref}`, @@ -135,7 +135,7 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { ); }; - wss.registerEvents('workers', { + wss.registerEvents('attempts:queue', { [CLAIM]: (ws, event) => pullClaim(state, ws, event), // is this part of the general workers pool, or part of the attempt? diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index bedc2b97d..3d1f171b6 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -69,6 +69,12 @@ type ServerOptions = { logger?: Logger; }; +// this is the websocket API +// basically a router +const createAPI = (ws) => { + // register events against the socket +}; + // for now all I wanna do is say hello const connectToLightning = (url: string, id: string) => { let socket = new Socket(url /*,{params: {userToken: "123"}}*/); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index edac92a85..b90f62fce 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -108,7 +108,7 @@ test.serial( new Promise(async (done) => { t.is(server.getQueueLength(), 0); - const channel = await join('workers'); + const channel = await join('attempts:queue'); // response is an array of attempt ids channel.push(CLAIM).receive('ok', (response) => { @@ -135,7 +135,7 @@ test.serial( // a) each worker has its own channel, so claims are handed out privately // b) we use the 'ok' status to return work in the response // this b pattern is much nicer - const channel = await join('workers'); + const channel = await join('attempts:queue'); // response is an array of attempt ids channel.push(CLAIM).receive('ok', (response) => { @@ -162,7 +162,7 @@ test.serial.skip( server.enqueueAttempt(attempt1); t.is(server.getQueueLength(), 3); - const channel = await join('workers'); + const channel = await join('attempts:queue'); // // response is an array of attempt ids // channel.push(CLAIM, { count: 3 }).receive('ok', (response) => { @@ -207,7 +207,7 @@ test.serial('get attempt data through the attempt channel', async (t) => { }); // TODO can't work out why this is failing - there's just no data in the response -test.serial.skip('get credential through the attempt channel', async (t) => { +test.serial.only('get credential through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addCredential('a', credentials['a']); From 546415b4af7e5427e01b6a55368726f5395a1c43 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 20 Sep 2023 15:28:23 +0200 Subject: [PATCH 070/232] rtm: add onMessage hook to sockets --- .../src/mock/lightning/socket-server.ts | 9 ++++---- .../test/mock/socket-server.test.ts | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index ffa79a69b..c1efa53ed 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -18,7 +18,7 @@ type PhoenixEvent = { type EventHandler = (event: string, payload: any) => void; -function createServer({ port = 8080, server, state } = {}) { +function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) { // console.log('ws listening on', port); const channels: Record> = {}; @@ -69,6 +69,7 @@ function createServer({ port = 8080, server, state } = {}) { wsServer.on('connection', function (ws: WS, req) { ws.on('message', function (data: string) { const evt = JSON.parse(data) as PhoenixEvent; + onMessage(evt); if (evt.topic) { // phx sends this info in each message @@ -108,10 +109,10 @@ function createServer({ port = 8080, server, state } = {}) { return new Promise((resolve) => { const listener = wsServer.listenToChannel( topic, - (e: string, payload: any) => { - if (e === event) { + (ws, e) => { + if (e.event === event) { listener.unsubscribe(); - resolve(payload); + resolve(event); } } ); diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index a91fae39c..4cd21a2ff 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -5,6 +5,7 @@ import createServer from '../../src/mock/lightning/socket-server'; let socket; let server; +let messages; const wait = (duration = 10) => new Promise((resolve) => { @@ -12,7 +13,8 @@ const wait = (duration = 10) => }); test.beforeEach(() => { - server = createServer(); + messages = []; + server = createServer({ onMessage: (evt) => messages.push(evt) }); socket = new phx.Socket('ws://localhost:8080'); socket.connect(); @@ -113,3 +115,20 @@ test.serial('wait for message', async (t) => { const result = await server.waitForMessage('x', 'hello'); t.truthy(result); }); + +test.serial.only('onMessage', (t) => { + return new Promise((done) => { + const channel = socket.channel('x', {}); + channel.join().receive('ok', async () => { + t.is(messages.length, 1) + t.is(messages[0].event, 'phx_join') + + channel.push('hello', { x: 1 }); + await server.waitForMessage('x', 'hello'); + t.is(messages.length, 2) + t.is(messages[1].event, 'hello') + done() + }) + }) + +}) \ No newline at end of file From 3b3409e9e4133127dbe35db7e66a0150ef419bb5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 20 Sep 2023 18:00:40 +0100 Subject: [PATCH 071/232] rtm: sketch out a testable way to implement the rtm server --- packages/rtm-server/src/events.ts | 3 + .../rtm-server/src/mock/lightning/api-dev.ts | 1 - packages/rtm-server/src/mock/lightning/api.ts | 4 - packages/rtm-server/src/start.ts | 6 ++ packages/rtm-server/src/work-loop.ts | 36 ++++--- packages/rtm-server/src/worker.ts | 84 ++++++++++++++++ packages/rtm-server/test/worker.test.ts | 98 +++++++++++++++++++ 7 files changed, 211 insertions(+), 21 deletions(-) create mode 100644 packages/rtm-server/src/worker.ts create mode 100644 packages/rtm-server/test/worker.test.ts diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 030a28ca8..0aae2ad4d 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -21,3 +21,6 @@ export const ATTEMPT_LOG = 'attempt:complete'; // level, namespace (job,runtime, // this should not happen - this is "could not execute" rather than "complete with errors" export const ATTEMPT_ERROR = 'attempt:error'; + +export const RUN_START = 'run:start'; +export const RUN_COMPLETE = 'run:complete'; diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index 42fa3132f..b6a579fd4 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -70,7 +70,6 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.getResult = (attemptId: string) => state.results[attemptId]?.state; - // TODO maybe onAttemptClaimed? or claim attempt? app.startAttempt = (attemptId: string) => api.startAttempt(attemptId); app.registerAttempt = (attempt: any) => { diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts index 31c85f690..de1bf6890 100644 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ b/packages/rtm-server/src/mock/lightning/api.ts @@ -137,14 +137,10 @@ export const createNewAPI = (state: ServerState, path: string, httpServer) => { wss.registerEvents('attempts:queue', { [CLAIM]: (ws, event) => pullClaim(state, ws, event), - - // is this part of the general workers pool, or part of the attempt? - // probably part of the attempt because we can control permissions }); const startAttempt = (attemptId) => { // mark the attempt as started on the server - // TODO right now this duplicates logic in the dev API state.pending[attemptId] = { status: 'started', }; diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index 10a42e188..d8a1e5aed 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -38,6 +38,12 @@ if (args.lightning === 'mock') { args.lightning = 'http://localhost:8888'; } +// TODO the rtm needs to take callbacks to load credential, and load state +// these in turn should utilise the websocket +// So either the server creates the runtime (which seems reasonable acutally?) +// Or the server calls a setCalbacks({ credential, state }) function on the RTM +// Each of these takes the attemptId as the firsdt argument +// credential and state will lookup the right channel const rtm = createRTM('rtm', { repoDir: args.repoDir }); logger.debug('RTM created'); diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts index 9606e42d7..e5f44dd5d 100644 --- a/packages/rtm-server/src/work-loop.ts +++ b/packages/rtm-server/src/work-loop.ts @@ -9,22 +9,26 @@ export default ( execute: (attempt: Attempt) => void ) => { const fetchWork = async () => { - // TODO what if this retuns like a 500? Server down? - const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { - method: 'POST', - body: JSON.stringify({ rtm_id: rtmId }), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); - if (result.body) { - const workflows = await result.json(); - if (workflows.length) { - workflows.forEach(execute); - return true; - } - } + // TODO this needs to use the socket now + // use getWithReply to claim the attempt + // then call execute just with the id + + // // TODO what if this retuns like a 500? Server down? + // const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { + // method: 'POST', + // body: JSON.stringify({ rtm_id: rtmId }), + // headers: { + // Accept: 'application/json', + // 'Content-Type': 'application/json', + // }, + // }); + // if (result.body) { + // const workflows = await result.json(); + // if (workflows.length) { + // workflows.forEach(execute); + // return true; + // } + // } // throw to backoff and try again throw new Error('backoff'); }; diff --git a/packages/rtm-server/src/worker.ts b/packages/rtm-server/src/worker.ts new file mode 100644 index 000000000..6c9f3e0f0 --- /dev/null +++ b/packages/rtm-server/src/worker.ts @@ -0,0 +1,84 @@ +// TODO not crazy about this file name +// This is the module responsible for interfacing between the Lightning websocket +// and the RTM +// It's the actual meat and potatoes of the implementation +// You can almost read this as a binding function and a bunch of handlers +// it isn't an actual worker, but a BRIDGE between a worker and lightning +import crypto from 'node:crypto'; + +import convertAttempt from './util/convert-attempt'; +// this managers the worker +//i would like functions to be testable, and I'd like the logic to be readable + +import { GET_ATTEMPT, RUN_START } from './events'; + +type Channel = any; // phx.Channel + +// TODO move to util +const getWithReply = (channel, event: string, payload?: any) => + new Promise((resolve) => { + channel.push(event, payload).receive('ok', (evt: any) => { + resolve(evt); + }); + // TODO handle errors amd timeouts too + }); + +function onJobStart(channel, state, jobId) { + // generate a run id + // write it to state + state.jobId = crypto.randomUUID(); + + // post the correct event to the lightning via websocket + // do we need to wait for a response? Well, not yet. + + channel.push(RUN_START, { + id: state.jobId, + job_id: jobId, + // input_dataclip_id what about this guy? + }); +} + +function onJobLog(channel, state) { + // we basically just forward the log to lightning + // but we also need to attach the log id +} + +// TODO actually I think this is prepare +export async function prepareAttempt(channel: Channel) { + // first we get the attempt body through the socket + const attemptBody = await getWithReply(channel, GET_ATTEMPT); + + // then we generate the execution plan + const plan = convertAttempt(attemptBody); + + return plan; + // difficulty: we need to tell the rtm how to callback for + // credentials and state (which should both be lazy and part of the run) + // I guess this is generic - given an attempt id I can lookup the channel and return this information + // then we call the excute function. Or return the promise and let someone else do that +} + +// These are the functions that lazy load data from lightning +// Is it appropriate first join the channel? Should there be some pooling? +async function loadState(ws, attemptId, stateId) {} + +async function loadCredential(ws, attemptId, stateId) {} + +// pass a web socket connected to the attempt channel +// this thing will do all the work +function execute(channel, rtm, attempt) { + // tracking state for this attempt + const state = { + runId: '', + }; + + // listen to rtm events + // what if I can do this + // this is super declarative + rtm.listen(attemptId, { + 'job-start': (evt) => onJobStart(ws, state, evt), + 'job-log': (evt) => onJobLog(ws, state), + }); + + rtm.execute(attempt); +} diff --git a/packages/rtm-server/test/worker.test.ts b/packages/rtm-server/test/worker.test.ts new file mode 100644 index 000000000..d25f104c4 --- /dev/null +++ b/packages/rtm-server/test/worker.test.ts @@ -0,0 +1,98 @@ +import test from 'ava'; +import { GET_ATTEMPT } from '../src/events'; +import { prepareAttempt } from '../src/worker'; +import { attempts } from './mock/data'; + +// This is a fake/mock websocket used by mocks + +const mockChannel = (callbacks) => { + return { + push: (event: string, payload: any) => { + // if a callback was registered, trigger it + // otherwise do nothing + + let result; + if (callbacks[event]) { + result = callbacks[event](payload); + } + + return { + receive: (status, callback) => { + // TODO maybe do this asynchronously? + callback(result); + }, + }; + }, + }; +}; + +test('mock channel: should invoke handler with payload', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + ping: (evt) => { + t.is(evt, 'abc'); + t.pass(); + done(); + }, + }); + + channel.push('ping', 'abc'); + }); +}); + +test('mock channel: invoke the ok handler with the callback result', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + ping: () => { + return 'pong!'; + }, + }); + + channel.push('ping', 'abc').receive('ok', (evt) => { + t.is(evt, 'pong!'); + t.pass(); + done(); + }); + }); +}); + +// TODO throw in the handler to get an error? + +test('prepareAttempt should get the attempt body', async (t) => { + const attempt = attempts['attempt-1']; + let didCallGetAttempt = false; + const channel = mockChannel({ + [GET_ATTEMPT]: () => { + // TODO should be no payload (or empty payload) + didCallGetAttempt = true; + }, + }); + + await prepareAttempt(channel, 'a1'); + t.true(didCallGetAttempt); +}); + +test('prepareAttempt should return an execution plan', async (t) => { + const attempt = attempts['attempt-1']; + + const channel = mockChannel({ + [GET_ATTEMPT]: () => attempt, + }); + + const plan = await prepareAttempt(channel, 'a1'); + t.deepEqual(plan, { + id: 'attempt-1', + jobs: [ + { + id: 'trigger', + configuration: 'a', + expression: 'fn(a => a)', + adaptor: '@openfn/language-common@1.0.0', + }, + ], + }); +}); + +test.skip('jobStart should emit the run id', () => {}); + +// TODO test the whole execute workflow From b9acbe7dc1ba949250b21909a43f72bc47750ce6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 21 Sep 2023 12:18:35 +0100 Subject: [PATCH 072/232] rtm: add extra events --- packages/rtm-server/src/worker.ts | 69 +++++++--- packages/rtm-server/test/worker.test.ts | 161 +++++++++++++++++++++++- 2 files changed, 212 insertions(+), 18 deletions(-) diff --git a/packages/rtm-server/src/worker.ts b/packages/rtm-server/src/worker.ts index 6c9f3e0f0..1c9ffc3b7 100644 --- a/packages/rtm-server/src/worker.ts +++ b/packages/rtm-server/src/worker.ts @@ -5,17 +5,25 @@ // You can almost read this as a binding function and a bunch of handlers // it isn't an actual worker, but a BRIDGE between a worker and lightning import crypto from 'node:crypto'; - +import phx from 'phoenix-channels'; +import { JSONLog } from '@openfn/logger'; import convertAttempt from './util/convert-attempt'; // this managers the worker //i would like functions to be testable, and I'd like the logic to be readable -import { GET_ATTEMPT, RUN_START } from './events'; +import { ATTEMPT_LOG, GET_ATTEMPT, RUN_COMPLETE, RUN_START } from './events'; +import { Attempt } from './types'; + +export type AttemptState = { + activeRun?: string; + activeJob?: string; + attempt: Attempt; +}; -type Channel = any; // phx.Channel +type Channel = typeof phx.Channel; // TODO move to util -const getWithReply = (channel, event: string, payload?: any) => +const getWithReply = (channel: Channel, event: string, payload?: any) => new Promise((resolve) => { channel.push(event, payload).receive('ok', (evt: any) => { resolve(evt); @@ -23,27 +31,55 @@ const getWithReply = (channel, event: string, payload?: any) => // TODO handle errors amd timeouts too }); -function onJobStart(channel, state, jobId) { +export function onJobStart( + channel: Channel, + state: AttemptState, + jobId: string +) { // generate a run id // write it to state - state.jobId = crypto.randomUUID(); + state.activeRun = crypto.randomUUID(); + state.activeJob = jobId; // post the correct event to the lightning via websocket // do we need to wait for a response? Well, not yet. channel.push(RUN_START, { - id: state.jobId, - job_id: jobId, + run_id: state.activeJob, + job_id: state.activeJob, + // input_dataclip_id what about this guy? }); } -function onJobLog(channel, state) { +export function onJobComplete( + channel: Channel, + state: AttemptState, + jobId: string +) { + channel.push(RUN_COMPLETE, { + run_id: state.activeJob, + job_id: state.activeJob, + // input_dataclip_id what about this guy? + }); + + delete state.activeRun; + delete state.activeJob; +} + +export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { // we basically just forward the log to lightning // but we also need to attach the log id + const evt = { + ...log, + attempt_id: state.attempt.id, + }; + if (state.activeRun) { + evt.run_id = state.activeRun; + } + channel.push(ATTEMPT_LOG, evt); } -// TODO actually I think this is prepare export async function prepareAttempt(channel: Channel) { // first we get the attempt body through the socket const attemptBody = await getWithReply(channel, GET_ATTEMPT); @@ -66,18 +102,21 @@ async function loadCredential(ws, attemptId, stateId) {} // pass a web socket connected to the attempt channel // this thing will do all the work -function execute(channel, rtm, attempt) { +function execute(channel: Channel, rtm, attempt) { // tracking state for this attempt - const state = { - runId: '', + const state: AttemptState = { + attempt, // keep this on the state so that anyone can access it }; // listen to rtm events // what if I can do this // this is super declarative - rtm.listen(attemptId, { + // TODO is there any danger of events coming through out of order? + // what if onJoblog takes 1 second to finish and before the runId is set, onJobLog comes through? + rtm.listen(attempt.id, { 'job-start': (evt) => onJobStart(ws, state, evt), - 'job-log': (evt) => onJobLog(ws, state), + 'job-complete': (evt) => onJobComplete(ws, state, evt), + 'job-log': (evt) => onJobLog(ws, state, evt), }); rtm.execute(attempt); diff --git a/packages/rtm-server/test/worker.test.ts b/packages/rtm-server/test/worker.test.ts index d25f104c4..6fdc74a6d 100644 --- a/packages/rtm-server/test/worker.test.ts +++ b/packages/rtm-server/test/worker.test.ts @@ -1,7 +1,18 @@ import test from 'ava'; -import { GET_ATTEMPT } from '../src/events'; -import { prepareAttempt } from '../src/worker'; +import { + GET_ATTEMPT, + RUN_START, + RUN_COMPLETE, + ATTEMPT_LOG, +} from '../src/events'; +import { + prepareAttempt, + onJobStart, + onJobComplete, + onJobLog, +} from '../src/worker'; import { attempts } from './mock/data'; +import { JSONLog } from '@openfn/logger'; // This is a fake/mock websocket used by mocks @@ -65,6 +76,7 @@ test('prepareAttempt should get the attempt body', async (t) => { [GET_ATTEMPT]: () => { // TODO should be no payload (or empty payload) didCallGetAttempt = true; + return attempt; }, }); @@ -93,6 +105,149 @@ test('prepareAttempt should return an execution plan', async (t) => { }); }); -test.skip('jobStart should emit the run id', () => {}); +test('jobStart should set a run id and active job on state', async (t) => { + const attempt = attempts['attempt-1']; + const jobId = 'job-1'; + + const state = { + attempt, + }; + + const channel = mockChannel({}); + + onJobStart(channel, state, jobId); + + t.is(state.activeJob, jobId); + t.truthy(state.activeRun); +}); + +test('jobStart should send a run:start event', async (t) => { + return new Promise((done) => { + const attempt = attempts['attempt-1']; + const jobId = 'job-1'; + + const state = { + attempt, + }; + + const channel = mockChannel({ + [RUN_START]: (evt) => { + t.is(evt.job_id, jobId); + t.truthy(evt.run_id); + + done(); + }, + }); + + onJobStart(channel, state, jobId); + }); +}); + +test('jobEnd should clear the run id and active job on state', async (t) => { + const attempt = attempts['attempt-1']; + const jobId = 'job-1'; + + const state = { + attempt, + activeJob: jobId, + activeRun: 'b', + }; + + const channel = mockChannel({}); + + onJobComplete(channel, state, jobId); + + t.falsy(state.activeJob); + t.falsy(state.activeRun); +}); + +test('jobComplete should send a run:complete event', async (t) => { + return new Promise((done) => { + const attempt = attempts['attempt-1']; + const jobId = 'job-1'; + + const state = { + attempt, + activeJob: jobId, + activeRun: 'b', + }; + + const channel = mockChannel({ + [RUN_COMPLETE]: (evt) => { + t.is(evt.job_id, jobId); + t.truthy(evt.run_id); + + done(); + }, + }); + + onJobComplete(channel, state, jobId); + }); +}); + +test('jobLog should should send a log event outside a run', async (t) => { + return new Promise((done) => { + const attempt = attempts['attempt-1']; + + const log: JSONLog = { + name: 'R/T', + level: 'info', + time: new Date().getTime(), + message: ['ping'], + }; + + const result = { + ...log, + attempt_id: attempt.id, + }; + + const state = { + attempt, + // No active run + }; + + const channel = mockChannel({ + [ATTEMPT_LOG]: (evt) => { + t.deepEqual(evt, result); + done(); + }, + }); + + onJobLog(channel, state, log); + }); +}); + +test('jobLog should should send a log event inside a run', async (t) => { + return new Promise((done) => { + const attempt = attempts['attempt-1']; + const jobId = 'job-1'; + + const log: JSONLog = { + name: 'R/T', + level: 'info', + time: new Date().getTime(), + message: ['ping'], + }; + + const state = { + attempt, + activeJob: jobId, + activeRun: 'b', + }; + + const channel = mockChannel({ + [ATTEMPT_LOG]: (evt) => { + t.truthy(evt.run_id); + t.deepEqual(evt.message, log.message); + t.is(evt.level, log.level); + t.is(evt.name, log.name); + t.is(evt.time, log.time); + done(); + }, + }); + + onJobLog(channel, state, log); + }); +}); // TODO test the whole execute workflow From ac560b32ced0f284d6812b173108d7ed90021b69 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 21 Sep 2023 14:10:04 +0100 Subject: [PATCH 073/232] rtm: bit of refactoring --- packages/rtm-server/src/events.ts | 2 + .../rtm-server/src/mock/runtime-manager.ts | 5 +- packages/rtm-server/src/worker.ts | 24 ++++--- packages/rtm-server/test/worker.test.ts | 62 +++++++++++++++---- 4 files changed, 69 insertions(+), 24 deletions(-) diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 0aae2ad4d..904b97437 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -24,3 +24,5 @@ export const ATTEMPT_ERROR = 'attempt:error'; export const RUN_START = 'run:start'; export const RUN_COMPLETE = 'run:complete'; + +// TODO I'd like to create payload type for each event, so that we have a central definition diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index fbc9daaf2..27c2f8b4f 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -84,7 +84,7 @@ function createMock( // TODO maybe lazy load the job from an id const { id, expression, configuration } = job; if (typeof configuration === 'string') { - // Fetch the credntial but do nothing with it + // Fetch the credential but do nothing with it // Maybe later we use it to assemble state await resolvers.credentials(configuration); } @@ -146,6 +146,9 @@ function createMock( once, execute, getStatus, + setResolvers: (r: LazyResolvers) => { + resolvers = r; + }, }; } diff --git a/packages/rtm-server/src/worker.ts b/packages/rtm-server/src/worker.ts index 1c9ffc3b7..d0a916f46 100644 --- a/packages/rtm-server/src/worker.ts +++ b/packages/rtm-server/src/worker.ts @@ -13,11 +13,12 @@ import convertAttempt from './util/convert-attempt'; import { ATTEMPT_LOG, GET_ATTEMPT, RUN_COMPLETE, RUN_START } from './events'; import { Attempt } from './types'; +import { ExecutionPlan } from '@openfn/runtime'; export type AttemptState = { activeRun?: string; activeJob?: string; - attempt: Attempt; + plan: ExecutionPlan; }; type Channel = typeof phx.Channel; @@ -72,7 +73,7 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { // but we also need to attach the log id const evt = { ...log, - attempt_id: state.attempt.id, + attempt_id: state.plan.id, }; if (state.activeRun) { evt.run_id = state.activeRun; @@ -82,7 +83,7 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { export async function prepareAttempt(channel: Channel) { // first we get the attempt body through the socket - const attemptBody = await getWithReply(channel, GET_ATTEMPT); + const attemptBody = (await getWithReply(channel, GET_ATTEMPT)) as Attempt; // then we generate the execution plan const plan = convertAttempt(attemptBody); @@ -102,10 +103,13 @@ async function loadCredential(ws, attemptId, stateId) {} // pass a web socket connected to the attempt channel // this thing will do all the work -function execute(channel: Channel, rtm, attempt) { +// TODO actually this now is a Workflow or Execution plan +// It's not an attempt anymore +export function execute(channel: Channel, rtm, plan: ExecutionPlan) { // tracking state for this attempt const state: AttemptState = { - attempt, // keep this on the state so that anyone can access it + //attempt, // keep this on the state so that anyone can access it + plan, }; // listen to rtm events @@ -113,11 +117,11 @@ function execute(channel: Channel, rtm, attempt) { // this is super declarative // TODO is there any danger of events coming through out of order? // what if onJoblog takes 1 second to finish and before the runId is set, onJobLog comes through? - rtm.listen(attempt.id, { - 'job-start': (evt) => onJobStart(ws, state, evt), - 'job-complete': (evt) => onJobComplete(ws, state, evt), - 'job-log': (evt) => onJobLog(ws, state, evt), + rtm.listen(plan.id, { + 'job-start': (evt) => onJobStart(plan, state, evt), + 'job-complete': (evt) => onJobComplete(plan, state, evt), + 'job-log': (evt) => onJobLog(plan, state, evt), }); - rtm.execute(attempt); + rtm.execute(plan); } diff --git a/packages/rtm-server/test/worker.test.ts b/packages/rtm-server/test/worker.test.ts index 6fdc74a6d..89ec061ce 100644 --- a/packages/rtm-server/test/worker.test.ts +++ b/packages/rtm-server/test/worker.test.ts @@ -10,9 +10,11 @@ import { onJobStart, onJobComplete, onJobLog, + execute, } from '../src/worker'; import { attempts } from './mock/data'; import { JSONLog } from '@openfn/logger'; +import createMockRTM from '../src/mock/runtime-manager'; // This is a fake/mock websocket used by mocks @@ -106,11 +108,11 @@ test('prepareAttempt should return an execution plan', async (t) => { }); test('jobStart should set a run id and active job on state', async (t) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const jobId = 'job-1'; const state = { - attempt, + plan, }; const channel = mockChannel({}); @@ -123,11 +125,11 @@ test('jobStart should set a run id and active job on state', async (t) => { test('jobStart should send a run:start event', async (t) => { return new Promise((done) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const jobId = 'job-1'; const state = { - attempt, + plan, }; const channel = mockChannel({ @@ -144,11 +146,11 @@ test('jobStart should send a run:start event', async (t) => { }); test('jobEnd should clear the run id and active job on state', async (t) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const jobId = 'job-1'; const state = { - attempt, + plan, activeJob: jobId, activeRun: 'b', }; @@ -163,11 +165,11 @@ test('jobEnd should clear the run id and active job on state', async (t) => { test('jobComplete should send a run:complete event', async (t) => { return new Promise((done) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const jobId = 'job-1'; const state = { - attempt, + plan, activeJob: jobId, activeRun: 'b', }; @@ -187,7 +189,7 @@ test('jobComplete should send a run:complete event', async (t) => { test('jobLog should should send a log event outside a run', async (t) => { return new Promise((done) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const log: JSONLog = { name: 'R/T', @@ -198,11 +200,11 @@ test('jobLog should should send a log event outside a run', async (t) => { const result = { ...log, - attempt_id: attempt.id, + attempt_id: plan.id, }; const state = { - attempt, + plan, // No active run }; @@ -219,7 +221,7 @@ test('jobLog should should send a log event outside a run', async (t) => { test('jobLog should should send a log event inside a run', async (t) => { return new Promise((done) => { - const attempt = attempts['attempt-1']; + const plan = { id: 'attempt-1' }; const jobId = 'job-1'; const log: JSONLog = { @@ -230,7 +232,7 @@ test('jobLog should should send a log event inside a run', async (t) => { }; const state = { - attempt, + plan, activeJob: jobId, activeRun: 'b', }; @@ -251,3 +253,37 @@ test('jobLog should should send a log event inside a run', async (t) => { }); // TODO test the whole execute workflow + +// run this against the mock - this just ensures that execute +// binds all the events +test.skip('execute should call all events', async (t) => { + const events = {}; + + const rtm = createMockRTM(); + + const channel = mockChannel({ + [ATTEMPT_LOG]: (evt) => { + events[ATTEMPT_LOG] = evt; + }, + }); + + const plan = { + id: 'attempt-1', + jobs: [ + { + id: 'trigger', + configuration: 'a', + expression: 'fn(a => a)', + adaptor: '@openfn/language-common@1.0.0', + }, + ], + }; + + const result = await execute(channel, rtm, plan); + + // check result is what we expect + + // Check that events were passed to the socket + // This is deliberately crude + t.truthy(events[ATTEMPT_LOG]); +}); From 6668814b11d9910c1b2103d1213118fa702c4b29 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 21 Sep 2023 14:42:11 +0100 Subject: [PATCH 074/232] rtm: add api to listen to workflow events in mock rtm --- .../rtm-server/src/mock/runtime-manager.ts | 51 ++++++++++---- .../test/mock/runtime-manager.test.ts | 69 +++++++++++++++++-- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 27c2f8b4f..4b26b09ba 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -28,27 +28,30 @@ export type RTMEvent = | 'workflow-error'; // ? export type JobStartEvent = { - id: string; // job id + workflowId: string; + jobId: string; runId: string; // run id. Not sure we need this. }; export type JobCompleteEvent = { - id: string; // job id - runId: string; // run id. Not sure we need this. + workflowId: string; + jobId: string; state: State; // do we really want to publish the intermediate events? Could be important, but also could be sensitive // I suppose at this level yes, we should publish it }; export type WorkflowStartEvent = { - id: string; // workflow id + workflowId: string; }; export type WorkflowCompleteEvent = { - id: string; // workflow id + workflowId: string; state?: object; error?: any; }; +// TODO log event optionally has a job id + let jobId = 0; const getNewJobId = () => `${++jobId}`; @@ -65,8 +68,12 @@ function createMock( ) { const activeWorkflows = {} as Record; const bus = new EventEmitter(); + const listeners: Record = {}; const dispatch = (type: RTMEvent, args?: any) => { + if (args.workflowId) { + listeners[args.workflowId]?.[type]?.(args); + } // TODO add performance metrics to every event? bus.emit(type, args); @@ -80,9 +87,19 @@ function createMock( bus.once(event, fn); }; - const executeJob = async (job: JobPlan, initialState = {}) => { + // Listens to events for a particular workflow/execution plan + // TODO: Listeners will be removed when the plan is complete (?) + const listen = ( + planId: string, + events: Record void> + ) => { + listeners[planId] = events; + }; + + const executeJob = async (workflowId, job: JobPlan, initialState = {}) => { // TODO maybe lazy load the job from an id const { id, expression, configuration } = job; + const jobId = id; if (typeof configuration === 'string') { // Fetch the credential but do nothing with it // Maybe later we use it to assemble state @@ -94,20 +111,24 @@ function createMock( // Get the job details from lightning // start instantly and emit as it goes - dispatch('job-start', { id, runId }); + dispatch('job-start', { workflowId, jobId }); let state = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { state = JSON.parse(expression); // What does this look like? Should be a logger object - dispatch('log', { message: ['Parsing expression as JSON state'] }); - dispatch('log', { message: [state] }); + dispatch('log', { + workflowId, + jobId, + message: ['Parsing expression as JSON state'], + }); + dispatch('log', { workflowId, jobId, message: [state] }); } catch (e) { // Do nothing, it's fine } - dispatch('job-complete', { id, runId, state }); + dispatch('job-complete', { workflowId, jobId, state }); return state; }; @@ -116,18 +137,21 @@ function createMock( // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = (xplan: ExecutionPlan) => { const { id, jobs } = xplan; + const workflowId = id; activeWorkflows[id!] = true; setTimeout(() => { - dispatch('workflow-start', { id }); + dispatch('workflow-start', { workflowId }); setTimeout(async () => { let state = {}; // Trivial job reducer in our mock for (const job of jobs) { - state = await executeJob(job, state); + state = await executeJob(id, job, state); } setTimeout(() => { delete activeWorkflows[id!]; - dispatch('workflow-complete', { id, state }); + dispatch('workflow-complete', { workflowId, state }); + // TODO on workflow complete we should maybe tidy the listeners? + // Doesn't really matter in the mock though }, 1); }, 1); }, 1); @@ -149,6 +173,7 @@ function createMock( setResolvers: (r: LazyResolvers) => { resolvers = r; }, + listen, }; } diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index a0745103f..7dcc0da53 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -42,7 +42,7 @@ test('Dispatch start events for a new workflow', async (t) => { rtm.execute(sampleWorkflow); const evt = await waitForEvent(rtm, 'workflow-start'); t.truthy(evt); - t.is(evt.id, 'w1'); + t.is(evt.workflowId, 'w1'); }); test('getStatus should report one active workflow', async (t) => { @@ -64,7 +64,7 @@ test('Dispatch complete events when a workflow completes', async (t) => { ); t.truthy(evt); - t.is(evt.id, 'w1'); + t.is(evt.workflowId, 'w1'); t.truthy(evt.state); }); @@ -74,8 +74,8 @@ test('Dispatch start events for a job', async (t) => { rtm.execute(sampleWorkflow); const evt = await waitForEvent(rtm, 'job-start'); t.truthy(evt); - t.is(evt.id, 'j1'); - t.truthy(evt.runId); + t.is(evt.workflowId, 'w1'); + t.is(evt.jobId, 'j1'); }); test('Dispatch complete events for a job', async (t) => { @@ -84,8 +84,8 @@ test('Dispatch complete events for a job', async (t) => { rtm.execute(sampleWorkflow); const evt = await waitForEvent(rtm, 'job-complete'); t.truthy(evt); - t.is(evt.id, 'j1'); - t.truthy(evt.runId); + t.is(evt.workflowId, 'w1'); + t.is(evt.jobId, 'j1'); t.truthy(evt.state); }); @@ -124,3 +124,60 @@ test('resolve credential before job-start if credential is a string', async (t) await waitForEvent(rtm, 'job-start'); t.true(didCallCredentials); }); + +test('listen to events', async (t) => { + const rtm = create(); + + const called = { + 'job-start': false, + 'job-complete': false, + log: false, + 'workflow-start': false, + 'workflow-complete': false, + }; + + rtm.listen(sampleWorkflow.id, { + 'job-start': ({ workflowId, jobId }) => { + called['job-start'] = true; + t.is(workflowId, sampleWorkflow.id); + t.is(jobId, sampleWorkflow.jobs[0].id); + }, + 'job-complete': ({ workflowId, jobId }) => { + called['job-complete'] = true; + t.is(workflowId, sampleWorkflow.id); + t.is(jobId, sampleWorkflow.jobs[0].id); + // TODO includes state? + }, + log: ({ workflowId, message }) => { + called['log'] = true; + t.is(workflowId, sampleWorkflow.id); + t.truthy(message); + }, + 'workflow-start': ({ workflowId }) => { + called['workflow-start'] = true; + t.is(workflowId, sampleWorkflow.id); + }, + 'workflow-complete': ({ workflowId }) => { + called['workflow-complete'] = true; + t.is(workflowId, sampleWorkflow.id); + }, + }); + + rtm.execute(sampleWorkflow); + await waitForEvent(rtm, 'workflow-complete'); + t.assert(Object.values(called).every((v) => v === true)); +}); + +test('only listen to events for the correct workflow', async (t) => { + const rtm = create(); + + rtm.listen('bobby mcgee', { + 'workflow-start': ({ workflowId }) => { + throw new Error('should not have called this!!'); + }, + }); + + rtm.execute(sampleWorkflow); + await waitForEvent(rtm, 'workflow-complete'); + t.pass(); +}); From 464c513b4c69fe63a075f9fa3b6424b569f3f04b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 21 Sep 2023 18:02:50 +0100 Subject: [PATCH 075/232] rtm: update lightning mock --- .../rtm-server/src/mock/lightning/api-dev.ts | 54 +++-- .../src/mock/lightning/api-sockets.ts | 143 ++++++++++++ packages/rtm-server/src/mock/lightning/api.ts | 211 ------------------ .../rtm-server/src/mock/lightning/server.ts | 31 ++- .../src/mock/lightning/socket-server.ts | 77 +++++-- .../rtm-server/test/mock/lightning.test.ts | 185 ++++----------- packages/rtm-server/test/socket-client.js | 30 +++ 7 files changed, 329 insertions(+), 402 deletions(-) create mode 100644 packages/rtm-server/src/mock/lightning/api-sockets.ts delete mode 100644 packages/rtm-server/src/mock/lightning/api.ts create mode 100644 packages/rtm-server/test/socket-client.js diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index b6a579fd4..7c58b7cd3 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -30,11 +30,25 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { state.credentials[id] = cred; }; + app.getCredential = (id: string) => state.credentials[id]; + app.addDataclip = (id: string, data: any) => { logger.info(`Add dataclip ${id}`); state.dataclips[id] = data; }; + app.getDataclip = (id: string) => state.dataclips[id]; + + app.enqueueAttempt = (attempt: Attempt) => { + state.attempts[attempt.id] = attempt; + state.results[attempt.id] = {}; + state.queue.push(attempt.id); + }; + + app.getAttempt = (id: string) => state.attempts[id]; + + app.getState = () => state; + // Promise which returns when a workflow is complete app.waitForResult = (attemptId: string) => { return new Promise((resolve) => { @@ -48,19 +62,6 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { }); }; - // Add an attempt to the queue - // TODO actually it shouldn't take an rtm id until it's pulled off the attempt - // Something feels off here - app.enqueueAttempt = (attempt: Attempt, rtmId: string = 'rtm') => { - logger.info(`Add Attempt ${attempt.id}`); - - state.results[attempt.id] = { - rtmId, - state: null, - }; - state.queue.push(attempt); - }; - app.reset = () => { state.queue = []; state.results = {}; @@ -72,6 +73,7 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.startAttempt = (attemptId: string) => api.startAttempt(attemptId); + // TODO probably remove? app.registerAttempt = (attempt: any) => { state.attempts[attempt.id] = attempt; }; @@ -91,17 +93,29 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { // Set up some rest endpoints // Note that these are NOT prefixed -const setupRestAPI = (app: DevApp, _state: ServerState, logger: Logger) => { +const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { const router = new Router(); router.post('/attempt', (ctx) => { - const data = ctx.request.body; - const rtmId = 'rtm'; // TODO include this in the body maybe? - if (!data.id) { - data.id = crypto.randomUUID(); - logger.info('Generating new id for incoming attempt:', data.id); + const attempt = ctx.request.body; + + logger.info('Adding new attempt to queue:', attempt.id); + + if (!attempt.id) { + attempt.id = crypto.randomUUID(); + logger.info('Generating new id for incoming attempt:', attempt.id); } - app.enqueueAttempt(data, rtmId); + + // convert credentials and dataclips + attempt.jobs.forEach((job) => { + if (job.credential) { + const cid = crypto.randomUUID(); + state.credentials[cid] = job.credential; + job.credential = cid; + } + }); + + app.enqueueAttempt(attempt); ctx.response.status = 200; }); diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts new file mode 100644 index 000000000..650660924 --- /dev/null +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -0,0 +1,143 @@ +import { WebSocketServer } from 'ws'; +import createLogger, { Logger } from '@openfn/logger'; + +import type { ServerState } from './server'; + +import { extractAttemptId } from './util'; + +import createPheonixMockSocketServer from './socket-server'; +import { CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP } from '../../events'; + +// this new API is websocket based +// Events map to handlers +// can I even implement this in JS? Not with pheonix anyway. hmm. +// dead at the first hurdle really. +// what if I do the server side mock in koa, can I use the pheonix client to connect? +const createSocketAPI = ( + state: ServerState, + path: string, + httpServer, + logger?: Logger +) => { + // set up a websocket server to listen to connections + // console.log('path', path); + const server = new WebSocketServer({ + server: httpServer, + + // Note: phoenix websocket will connect to /websocket + path: path ? `${path}/websocket` : undefined, + }); + + // pass that through to the phoenix mock + const wss = createPheonixMockSocketServer({ + server, + state, + logger: logger && createLogger('PHX', { level: 'debug' }), + }); + + // TODO + // 1) Need to improve the abstraction of these, make messages easier to send + // 2) Also need to look at closures - I'd like a declarative central API + // the need to call startAttempt makes things a bit harder + + // pull claim will try and pull a claim off the queue, + // and reply with the response + // the reply ensures that only the calling worker will get the attempt + const pullClaim = (state, ws, evt) => { + const { ref, topic } = evt; + const { queue } = state; + let count = 1; + + const payload = { + status: 'ok', + response: [], + }; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + const next = queue.shift(); + payload.response.push(next); + count -= 1; + + startAttempt(next.id); + } + if (payload.response.length) { + logger?.info(`Claiming ${payload.response.length} attempts`); + } else { + logger?.info('No claims (queue empty)'); + } + + ws.reply({ ref, topic, payload }); + }; + + const getAttempt = (state, ws, evt) => { + const { ref, topic } = evt; + const attemptId = extractAttemptId(topic); + const attempt = state.attempts[attemptId]; + + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + response: attempt, + }, + }); + }; + + const getCredential = (state, ws, evt) => { + const { ref, topic, payload, event } = evt; + const response = state.credentials[payload.id]; + // console.log(topic, event, response); + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + response, + }, + }); + }; + + const getDataclip = (state, ws, evt) => { + const { ref, topic, payload, event } = evt; + const response = state.dataclips[payload.id]; + console.log(response); + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + response, + }, + }); + }; + + wss.registerEvents('attempts:queue', { + [CLAIM]: (ws, event) => pullClaim(state, ws, event), + }); + + const startAttempt = (attemptId) => { + // mark the attempt as started on the server + state.pending[attemptId] = { + status: 'started', + }; + + // TODO do all these need extra auth, or is auth granted + // implicitly by channel membership? + // Right now the socket gets access to all server state + // But this is just a mock - Lightning can impose more restrictions if it wishes + wss.registerEvents(`attempt:${attemptId}`, { + [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), + [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), + [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), + }); + }; + + return { + startAttempt, + }; +}; + +export default createSocketAPI; diff --git a/packages/rtm-server/src/mock/lightning/api.ts b/packages/rtm-server/src/mock/lightning/api.ts deleted file mode 100644 index de1bf6890..000000000 --- a/packages/rtm-server/src/mock/lightning/api.ts +++ /dev/null @@ -1,211 +0,0 @@ -import Router from '@koa/router'; -import Socket, { WebSocketServer } from 'ws'; -import { - unimplemented, - createListNextJob, - createClaim, - createGetCredential, - createLog, - createComplete, -} from './middleware'; -import type { ServerState } from './server'; - -import { API_PREFIX } from './server'; -import { extractAttemptId } from './util'; - -import createPheonixMockSocketServer from './socket-server'; -import { CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP } from '../../events'; - -interface RTMBody { - rtm_id: string; -} - -export interface FetchNextBody extends RTMBody {} - -export interface AttemptCompleteBody extends RTMBody { - state: any; // JSON state object (undefined? null?) -} - -// this new API is websocket based -// Events map to handlers -// can I even implement this in JS? Not with pheonix anyway. hmm. -// dead at the first hurdle really. -// what if I do the server side mock in koa, can I use the pheonix client to connect? -export const createNewAPI = (state: ServerState, path: string, httpServer) => { - // set up a websocket server to listen to connections - // console.log('path', path); - const server = new WebSocketServer({ - server: httpServer, - - // Note: phoenix websocket will connect to /websocket - path: path ? `${path}/websocket` : undefined, - }); - - // pass that through to the phoenix mock - const wss = createPheonixMockSocketServer({ server, state }); - - // TODO - // 1) Need to improve the abtraction of these, make messages easier to send - // 2) Also need to look at closures - I'd like a declarative central API - // the need to call startAttempt makes things a bit harder - - // pull claim will try and pull a claim off the queue, - // and reply with the response - // the reply ensures that only the calling worker will get the attempt - const pullClaim = (state, ws, evt) => { - const { ref, topic } = evt; - const { queue } = state; - let count = 1; - - const payload = { - status: 'ok', - response: [], - }; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - const next = queue.shift(); - payload.response.push(next.id); - count -= 1; - - startAttempt(next.id); - } - - ws.send( - JSON.stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload, - }) - ); - }; - - const getAttempt = (state, ws, evt) => { - const { ref, topic } = evt; - const attemptId = extractAttemptId(topic); - const attempt = state.attempts[attemptId]; - - ws.send( - JSON.stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload: { - status: 'ok', - response: attempt, - }, - }) - ); - }; - - const getCredential = (state, ws, evt) => { - const { ref, topic, payload, event } = evt; - const response = state.credentials[payload.id]; - // console.log(topic, event, response); - ws.send( - JSON.stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload: { - status: 'ok', - response, - }, - }) - ); - }; - - const getDataclip = (state, ws, evt) => { - console.log(' getDataClip'); - const { ref, topic, payload, event } = evt; - const response = state.dataclips[payload.id]; - console.log(response); - ws.send( - JSON.stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload: { - status: 'ok', - response, - }, - }) - ); - }; - - wss.registerEvents('attempts:queue', { - [CLAIM]: (ws, event) => pullClaim(state, ws, event), - }); - - const startAttempt = (attemptId) => { - // mark the attempt as started on the server - state.pending[attemptId] = { - status: 'started', - }; - - // TODO do all these need extra auth, or is auth granted - // implicitly by channel membership? - // Right now the socket gets access to all server state - // But this is just a mock - Lightning can impose more restrictions if it wishes - wss.registerEvents(`attempt:${attemptId}`, { - [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), - [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), - [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), - }); - }; - - return { - startAttempt, - }; -}; - -// Note that this API is hosted at api/1 -// Deprecated -export default (state: ServerState) => { - const router = new Router({ prefix: API_PREFIX }); - // Basically all requests must include an rtm_id (And probably later a security token) - // TODO actually, is this an RTM Id or an RTM Server id? - - // POST attempts/next - // Removes Attempts from the queue and returns them to the caller - // Lightning should track who has each attempt - // 200 - return an array of pending attempts - // 204 - queue empty (no body) - router.post('/attempts/next', createFetchNextJob(state)); - - // GET credential/:id - // Get a credential - // 200 - return a credential object - // 404 - credential not found - router.get('/credential/:id', createGetCredential(state)); - - // Notify for a batch of job logs - // [{ rtm_id, logs: ['hello world' ] }] - // TODO this could use a websocket to handle the high volume of logs - router.post('/attempts/log/:id', createLog(state)); - - // Notify an attempt has finished - // Could be error or success state - // If a complete comes in from an unexpected source (ie a timed out job), this should throw - // state and rtm_id should be in the payload - // { rtm,_id, state } | { rtmId, error } - router.post('/attempts/complete/:id', createComplete(state)); - - // TODO i want this too: confirm that an attempt has started - router.post('/attempts/start/:id', () => {}); - - // Listing APIs - these list details without changing anything - router.get('/attempts/next', createListNextJob(state)); // ?count=1 - router.get('/attempts/:id', unimplemented); - router.get('/attempts/done', unimplemented); // ?project=pid - router.get('/attempts/active', unimplemented); - - router.get('/credential/:id', unimplemented); - - router.get('/workflows', unimplemented); - router.get('/workflows/:id', unimplemented); - - return router.routes(); -}; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index cc7a93ab9..7c8a3cfa5 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -5,10 +5,15 @@ import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import websockify from 'koa-websocket'; import route from 'koa-route'; -import { createMockLogger, LogLevel, Logger } from '@openfn/logger'; +import createLogger, { + createMockLogger, + LogLevel, + Logger, +} from '@openfn/logger'; import createServer from './socket-server'; -import createAPI, { createNewAPI } from './api'; +import createAPI from './api'; +import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; @@ -56,13 +61,25 @@ const createLightningServer = (options: LightningOptions = {}) => { const app = new Koa(); app.use(bodyParser()); - const server = app.listen(options.port || 8888); + const port = options.port || 8888; + const server = app.listen(port); + logger.info('Listening on ', port); // Setup the websocket API - const api = createNewAPI(state, '/api', server); - - const klogger = koaLogger((str) => logger.debug(str)); - app.use(klogger); + const api = createWebSocketAPI( + state, + '/api', + server, + options.logger && logger + ); + + // Only create a http logger if there's a top-level logger passed + // This is a bit flaky really but whatever + if (options.logger) { + const httpLogger = createLogger('HTTP', { level: 'debug' }); + const klogger = koaLogger((str) => httpLogger.debug(str)); + app.use(klogger); + } // Mock API endpoints // TODO should we keep the REST interface for local debug? diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index c1efa53ed..15af240be 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -18,8 +18,14 @@ type PhoenixEvent = { type EventHandler = (event: string, payload: any) => void; -function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) { - // console.log('ws listening on', port); +function createServer({ + port = 8080, + server, + state, + logger, + onMessage = () => {}, +} = {}) { + logger?.info('pheonix mock websocket server listening on', port); const channels: Record> = {}; const wsServer = @@ -31,14 +37,12 @@ function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) const events = { // testing (TODO shouldn't this be in a specific channel?) ping: (ws, { topic, ref }) => { - ws.send( - JSON.stringify({ - topic, - ref, - event: 'pong', - payload: {}, - }) - ); + ws.sendJSON({ + topic, + ref, + event: 'pong', + payload: {}, + }); }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket phx_join: (ws, { event, topic, ref }) => { @@ -53,20 +57,44 @@ function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) response = 'invalid_attempt'; } } + ws.reply({ + topic, + payload: { status, response }, + ref, + }); + }, + }; + + wsServer.on('connection', function (ws: WS, req) { + // TODO need to be logging here really + logger?.info('new connection'); + + ws.reply = ({ ref, topic, payload }: PhoenixEvent) => { + logger?.debug( + `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) + ); ws.send( JSON.stringify({ - // here is the magic reply event - // see channel.replyEventName event: `chan_reply_${ref}`, + ref, topic, - payload: { status, response }, + payload, + }) + ); + }; + + ws.sendJSON = ({ event, ref, topic, payload }: PhoenixEvent) => { + logger?.debug(`<< [${topic}] ${event} ` + JSON.stringify(payload)); + ws.send( + JSON.stringify({ + event, ref, + topic, + payload, }) ); - }, - }; + }; - wsServer.on('connection', function (ws: WS, req) { ws.on('message', function (data: string) { const evt = JSON.parse(data) as PhoenixEvent; onMessage(evt); @@ -75,6 +103,10 @@ function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) // phx sends this info in each message const { topic, event, payload, ref } = evt; + logger?.debug( + `>> [${topic}] ${event} ${ref} :: ${JSON.stringify(payload)}` + ); + if (events[event]) { // handle system/phoenix events events[event](ws, { topic, payload, ref }); @@ -107,15 +139,12 @@ function createServer({ port = 8080, server, state, onMessage = () => {} } = {}) wsServer.waitForMessage = (topic: Topic, event: string) => { return new Promise((resolve) => { - const listener = wsServer.listenToChannel( - topic, - (ws, e) => { - if (e.event === event) { - listener.unsubscribe(); - resolve(event); - } + const listener = wsServer.listenToChannel(topic, (ws, e) => { + if (e.event === event) { + listener.unsubscribe(); + resolve(event); } - ); + }); }); }; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index b90f62fce..bce4c3c6a 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -1,6 +1,5 @@ import test from 'ava'; import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; -import { createMockLogger } from '@openfn/logger'; import phx from 'phoenix-channels'; @@ -11,16 +10,10 @@ import { GET_CREDENTIAL, GET_DATACLIP, } from '../../src/events'; +import type { Attempt } from '../../src/types'; const endpoint = 'ws://localhost:7777/api'; -const baseUrl = `http://localhost:7777${API_PREFIX}`; - -const sleep = (duration = 10) => - new Promise((resolve) => { - setTimeout(resolve, duration); - }); - let server; let client; @@ -59,17 +52,55 @@ const join = (channelName: string): Promise => }); }); -const get = (path: string) => fetch(`${baseUrl}/${path}`); -const post = (path: string, data: any) => - fetch(`${baseUrl}/${path}`, { +// Test some dev hooks +// enqueue attempt should add id to the queue and register the state, credentials and body +test.serial('should setup an attempt at /POST /attempt', async (t) => { + const state = server.getState(); + + t.is(Object.keys(state.credentials).length, 0); + t.is(Object.keys(state.attempts).length, 0); + t.is(Object.keys(state.attempts).length, 0); + + const attempt: Attempt = { + id: 'a', + triggers: [], + jobs: [ + { + body: 'abc', + dataclip: {}, // not sure how this will work on the attempt yet + credential: { + // this will be converted into a string for lazy loading + user: 'john', + password: 'rambo', + }, + }, + ], + edges: [], + }; + + await fetch('http://localhost:7777/attempt', { method: 'POST', - body: JSON.stringify(data), + body: JSON.stringify(attempt), headers: { Accept: 'application/json', 'Content-Type': 'application/json', }, }); + const newState = server.getState(); + t.is(Object.keys(newState.attempts).length, 1); + const a = server.getAttempt('a'); + t.truthy(a); + t.is(server.getQueueLength(), 1); + + t.is(Object.keys(newState.credentials).length, 1); + + const job = a.jobs[0]; + t.assert(typeof job.credential === 'string'); + const c = server.getCredential(job.credential); + t.is(c.user, 'john'); +}); + test.serial('provide a phoenix websocket at /api', (t) => { // client should be connected before this test runs t.is(client.connectionState(), 'open'); @@ -187,7 +218,7 @@ test.serial('create a channel for an attempt', async (t) => { }); test.serial('reject channels for attempts that are not started', async (t) => { - await t.throwsAsync(() => join('attempt:wibble'), { + await t.throwsAsync(() => join('attempt:xyz'), { message: 'invalid_attempt', }); }); @@ -199,7 +230,6 @@ test.serial('get attempt data through the attempt channel', async (t) => { const channel = await join(`attempt:${attempt1.id}`); channel.push(GET_ATTEMPT).receive('ok', (p) => { - console.log('attempt', p); t.deepEqual(p, attempt1); done(); }); @@ -207,7 +237,7 @@ test.serial('get attempt data through the attempt channel', async (t) => { }); // TODO can't work out why this is failing - there's just no data in the response -test.serial.only('get credential through the attempt channel', async (t) => { +test.serial.skip('get credential through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addCredential('a', credentials['a']); @@ -233,128 +263,3 @@ test.serial.skip('get dataclip through the attempt channel', async (t) => { }); }); }); - -// TODO not going to bother testing attempt_ logs, they're a bit more passive in the process -// Lightning doesn't really care about them -// should we acknowledge them maybe though? - -test.serial.skip('POST /attempts/log - should return 200', async (t) => { - server.enqueueAttempt(attempt1); - const { status } = await post('attempts/log/attempt-1', { - rtm_id: 'rtm', - logs: [{ message: 'hello world' }], - }); - t.is(status, 200); -}); - -test.serial.skip( - 'POST /attempts/log - should return 400 if no rtm_id', - async (t) => { - const { status } = await post('attempts/log/attempt-1', { - rtm_id: 'rtm', - logs: [{ message: 'hello world' }], - }); - t.is(status, 400); - } -); - -test.serial.skip( - 'POST /attempts/log - should echo to event emitter', - async (t) => { - server.enqueueAttempt(attempt1); - let evt; - let didCall = false; - - server.once('log', (e) => { - didCall = true; - evt = e; - }); - - const { status } = await post('attempts/log/attempt-1', { - rtm_id: 'rtm', - logs: [{ message: 'hello world' }], - }); - t.is(status, 200); - t.true(didCall); - - t.truthy(evt); - t.is(evt.id, 'attempt-1'); - t.deepEqual(evt.logs, [{ message: 'hello world' }]); - } -); - -test.serial.skip('POST /attempts/complete - return final state', async (t) => { - server.enqueueAttempt(attempt1); - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - x: 10, - }, - }); - t.is(status, 200); - const result = server.getResult('attempt-1'); - t.deepEqual(result, { x: 10 }); -}); - -test.serial.skip( - 'POST /attempts/complete - reject if unknown rtm', - async (t) => { - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - x: 10, - }, - }); - t.is(status, 400); - t.falsy(server.getResult('attempt-1')); - } -); - -test.serial.skip( - 'POST /attempts/complete - reject if unknown workflow', - async (t) => { - server.enqueueAttempt({ id: 'b' }, 'rtm'); - - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - x: 10, - }, - }); - - t.is(status, 400); - t.falsy(server.getResult('attempt-1')); - } -); - -test.serial.skip( - 'POST /attempts/complete - echo to event emitter', - async (t) => { - server.enqueueAttempt(attempt1); - let evt; - let didCall = false; - - server.once('attempt-complete', (e) => { - didCall = true; - evt = e; - }); - - const { status } = await post('attempts/complete/attempt-1', { - rtm_id: 'rtm', - state: { - data: { - answer: 42, - }, - }, - }); - t.is(status, 200); - t.true(didCall); - - t.truthy(evt); - t.is(evt.rtm_id, 'rtm'); - t.is(evt.workflow_id, 'attempt-1'); - t.deepEqual(evt.state, { data: { answer: 42 } }); - } -); - -// test lightning should get the finished state through a helper API diff --git a/packages/rtm-server/test/socket-client.js b/packages/rtm-server/test/socket-client.js new file mode 100644 index 000000000..fd3f83dbf --- /dev/null +++ b/packages/rtm-server/test/socket-client.js @@ -0,0 +1,30 @@ +// this is a standalone test script +// run from the commandline ie `node test/socket-client.js` +import phx from 'phoenix-channels'; + +const endpoint = 'ws://localhost:8888/api'; + +console.log('connecting to socket at ', endpoint); +const socket = new phx.Socket(endpoint); + +socket.onOpen(() => { + console.log('socket open!'); + + const channel = socket.channel('attempts:queue'); + channel.join().receive('ok', () => { + console.log('connected to attempts queue'); + + channel.on('pong', () => { + console.log('received pong!'); + }); + + channel.push('ping'); + }); + + setInterval(() => { + console.log('requesting work...'); + channel.push('attempts:claim'); + }, 500); +}); + +socket.connect(); From ba750b9d4aee218bc4094701e4e8e53e8ff6acc4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 22 Sep 2023 15:46:17 +0100 Subject: [PATCH 076/232] rtm: setup unit tests around server connection and work loop --- .../src/mock/lightning/middleware.ts | 1 + packages/rtm-server/src/server.ts | 68 +++++++++-- .../rtm-server/src/util/try-with-backoff.ts | 36 +++++- packages/rtm-server/test/server.test.ts | 106 +++++++++++++++--- packages/rtm-server/test/util.test.ts | 84 ++++++++++++++ packages/rtm-server/test/util.ts | 51 +++++++++ .../test/util/try-with-backoff.test.ts | 38 ++++++- packages/rtm-server/test/worker.test.ts | 51 --------- 8 files changed, 356 insertions(+), 79 deletions(-) create mode 100644 packages/rtm-server/test/util.test.ts diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts index 160b52cf1..dd555bc7c 100644 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ b/packages/rtm-server/src/mock/lightning/middleware.ts @@ -1,3 +1,4 @@ +/// Thhis is basically deprecated import Koa from 'koa'; import type { ServerState } from './server'; import { AttemptCompleteBody } from './api'; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 3d1f171b6..38d2bf030 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -10,12 +10,15 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; +import phx from 'phoenix-channels'; +import { createMockLogger, Logger } from '@openfn/logger'; import createAPI from './api'; -import startWorkLoop from './work-loop'; +// import startWorkLoop from './work-loop'; +import { tryWithBackoff } from './util'; import convertAttempt from './util/convert-attempt'; import { Attempt } from './types'; -import { createMockLogger, Logger } from '@openfn/logger'; +import { CLAIM } from './events'; const postResult = async ( rtmId: string, @@ -75,12 +78,61 @@ const createAPI = (ws) => { // register events against the socket }; -// for now all I wanna do is say hello -const connectToLightning = (url: string, id: string) => { - let socket = new Socket(url /*,{params: {userToken: "123"}}*/); - socket.connect(); - const channel = socket.channel('worker'); - channel.push('hello'); +// This will open up a websocket channel to lightning +// TODO auth +export const connectToLightning = ( + endpoint: string, + id: string, + Socket = phx.Socket +) => { + return new Promise((done) => { + let socket = new Socket(endpoint /*,{params: {userToken: "123"}}*/); + socket.connect(); + + // join the queue channel + const channel = socket.channel('attempts:queue'); + + channel.join().receive('ok', () => { + done(channel); + }); + }); +}; + +// TODO this needs to return some kind of cancel function +export const startWorkloop = (channel, execute, delay = 100) => { + let promise; + let cancelled = false; + + const request = () => { + channel.push(CLAIM).receive('ok', (attempts) => { + if (!attempts.length) { + // throw to backoff and try again + throw new Error('backoff'); + } + attempts.forEach((attempt) => { + execute(attempt); + }); + }); + }; + + const workLoop = () => { + if (!cancelled) { + promise = tryWithBackoff(request, { timeout: delay }); + // promise.then(workLoop).catch(() => { + // // this means the backoff expired + // // which right now it won't ever do + // // but what's the plan? + // // log and try again I guess? + // workLoop(); + // }); + } + }; + workLoop(); + + return () => { + cancelled = true; + promise.cancel(); + }; }; function createServer(rtm: any, options: ServerOptions = {}) { diff --git a/packages/rtm-server/src/util/try-with-backoff.ts b/packages/rtm-server/src/util/try-with-backoff.ts index b3da9d864..718484573 100644 --- a/packages/rtm-server/src/util/try-with-backoff.ts +++ b/packages/rtm-server/src/util/try-with-backoff.ts @@ -3,6 +3,11 @@ type Options = { maxAttempts?: number; maxBackoff?: number; timeout?: number; + isCancelled?: () => boolean; +}; + +type CancelablePromise = Promise & { + cancel: () => void; }; const MAX_BACKOFF = 1000 * 60; @@ -10,7 +15,7 @@ const MAX_BACKOFF = 1000 * 60; // This function will try and call its first argument every {opts.timeout|100}ms // If the function throws, it will "backoff" and try again a little later // Right now it's a bit of a sketch, but it sort of works! -const tryWithBackoff = (fn: any, opts: Options = {}) => { +const tryWithBackoff = (fn: any, opts: Options = {}): CancelablePromise => { if (!opts.timeout) { opts.timeout = 100; } @@ -21,28 +26,51 @@ const tryWithBackoff = (fn: any, opts: Options = {}) => { timeout = timeout; attempts = attempts; - return new Promise(async (resolve, reject) => { + let cancelled = false; + + if (!opts.isCancelled) { + // Keep the top-level cancel flag in scope + // This way nested promises will still use the same flag and let + // themselves be cancelled + opts.isCancelled = () => cancelled; + } + + const promise = new Promise(async (resolve, reject) => { try { await fn(); resolve(); } catch (e) { + if (opts.isCancelled!()) { + return resolve(); + } + if (!isNaN(maxAttempts as any) && attempts >= (maxAttempts as number)) { return reject(new Error('max attempts exceeded')); } // failed? No problem, we'll back off and try again - // TODO update opts - // TODO is this gonna cause a crazy promise chain? setTimeout(() => { + if (opts.isCancelled!()) { + return resolve(); + } const nextOpts = { maxAttempts, attempts: attempts + 1, timeout: Math.min(MAX_BACKOFF, timeout * 1.2), + isCancelled: opts.isCancelled, }; tryWithBackoff(fn, nextOpts).then(resolve).catch(reject); }, timeout); } }); + + // allow the try to be cancelled + // We can't cancel the active in-flight promise but we can prevent the callback + (promise as CancelablePromise).cancel = () => { + cancelled = true; + }; + + return promise as CancelablePromise; }; export default tryWithBackoff; diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index 8f2637e54..938c3ee26 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,14 +1,17 @@ import test from 'ava'; import WebSocket, { WebSocketServer } from 'ws'; -import createServer from '../src/server'; +import createServer, { connectToLightning, startWorkloop } from '../src/server'; import createMockRTM from '../src/mock/runtime-manager'; +import { mockChannel, mockSocket, sleep } from './util'; +import { CLAIM } from '../src/events'; // Unit tests against the RTM web server // I don't think there will ever be much here because the server is mostly a pull let rtm; let server; +let cancel; const url = 'http://localhost:7777'; @@ -17,7 +20,8 @@ test.beforeEach(() => { }); test.afterEach(() => { - server.close(); // whatever + cancel?.(); // cancel any workloops + server?.close(); // whatever }); test.skip('healthcheck', async (t) => { @@ -28,23 +32,95 @@ test.skip('healthcheck', async (t) => { t.is(body, 'OK'); }); -test('connects to websocket', (t) => { - let didSayHello; +// Not a very thorough test +test('connects to lightning', async (t) => { + await connectToLightning('www', 'rtm', mockSocket); + t.pass(); - const wss = new WebSocketServer({ - port: 8080, + // TODO connections to hte same socket.channel should share listners, so I think I can test the channel +}); + +test('workloop can be cancelled', async (t) => { + let count = 0; + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + count++; + cancel(); + }, }); - wss.on('message', () => { - didSayHello = true; + + cancel = startWorkloop(channel, () => {}, 1); + + await sleep(100); + // A quirk of how cancel works is that the loop will be called a few times + t.assert(count <= 5); +}); + +test('workloop sends the attempts:claim event', (t) => { + return new Promise((done) => { + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + t.pass(); + done(); + }, + }); + cancel = startWorkloop(channel, () => {}); }); +}); - rtm = createMockRTM(); - server = createServer(rtm, { - port: 7777, - lightning: 'ws://localhost:8080', - // TODO what if htere's some kind of onready hook? - // TODO also we'll need some utility like waitForEvent +test('workloop sends the attempts:claim event several times ', (t) => { + return new Promise((done) => { + let cancel; + let count = 0; + const channel = mockChannel({ + [CLAIM]: () => { + count++; + if (count === 5) { + t.pass(); + done(); + } + }, + }); + cancel = startWorkloop(channel, () => {}); }); +}); + +test('workloop calls execute if attempts:claim returns attempts', (t) => { + return new Promise((done) => { + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + return [{ id: 'a' }]; + }, + }); - t.true(didSayHello); + cancel = startWorkloop(channel, (attempt) => { + t.deepEqual(attempt, { id: 'a' }); + t.pass(); + done(); + }); + }); }); + +// test('connects to websocket', (t) => { +// let didSayHello; + +// const wss = new WebSocketServer({ +// port: 8080, +// }); +// wss.on('message', () => { +// didSayHello = true; +// }); + +// rtm = createMockRTM(); +// server = createServer(rtm, { +// port: 7777, +// lightning: 'ws://localhost:8080', +// // TODO what if htere's some kind of onready hook? +// // TODO also we'll need some utility like waitForEvent +// }); + +// t.true(didSayHello); +// }); diff --git a/packages/rtm-server/test/util.test.ts b/packages/rtm-server/test/util.test.ts new file mode 100644 index 000000000..d03a8d2ad --- /dev/null +++ b/packages/rtm-server/test/util.test.ts @@ -0,0 +1,84 @@ +import test from 'ava'; +import { mockSocket, mockChannel } from './util'; + +test('mock channel: join', (t) => { + return new Promise((done) => { + const channel = mockChannel(); + t.assert(channel.hasOwnProperty('push')); + t.assert(channel.hasOwnProperty('join')); + + channel.join().receive('ok', () => { + t.pass(); + done(); + }); + }); +}); + +test('mock channel: should invoke handler with payload', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + ping: (evt) => { + t.is(evt, 'abc'); + t.pass(); + done(); + }, + }); + + channel.push('ping', 'abc'); + }); +}); + +test('mock channel: invoke the ok handler with the callback result', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + ping: () => { + return 'pong!'; + }, + }); + + channel.push('ping', 'abc').receive('ok', (evt) => { + t.is(evt, 'pong!'); + t.pass(); + done(); + }); + }); +}); + +test('mock channel: listen to event', (t) => { + return new Promise((done) => { + const channel = mockChannel(); + + channel.on('ping', () => { + t.pass(); + done(); + }); + + channel.push('ping'); + }); +}); + +test('mock socket: connect', (t) => { + return new Promise((done) => { + const socket = mockSocket(); + + // this is a noop + socket.connect(); + t.pass('connected'); + done(); + }); +}); + +test('mock socket: connect to channel', (t) => { + return new Promise((done) => { + const socket = mockSocket(); + + const channel = socket.channel('abc'); + t.assert(channel.hasOwnProperty('push')); + t.assert(channel.hasOwnProperty('join')); + + channel.join().receive('ok', () => { + t.pass(); + done(); + }); + }); +}); diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index 839ef533f..04f5cc806 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -24,3 +24,54 @@ export const waitForEvent = (rtm, eventName) => resolve(e); }); }); + +export const sleep = (delay = 100) => + new Promise((resolve) => { + setTimeout(resolve, delay); + }); + +export const mockChannel = (callbacks = {}) => { + const c = { + on: (event, fn) => { + // TODO support multiple callbacks + callbacks[event] = fn; + }, + push: (event: string, payload?: any) => { + // if a callback was registered, trigger it + // otherwise do nothing + let result; + if (callbacks[event]) { + result = callbacks[event](payload); + } + + return { + receive: (status, callback) => { + // TODO maybe do this asynchronously? + callback(result); + }, + }; + }, + join: () => { + return { + receive: (status, callback) => { + callback(); + }, + }; + }, + }; + return c; +}; + +export const mockSocket = () => { + const channels = {}; + return { + connect: () => { + // noop + // TODO maybe it'd be helpful to throw if the channel isn't connected? + }, + channel: (topic: string) => { + channels[topic] = mockChannel(); + return channels[topic]; + }, + }; +}; diff --git a/packages/rtm-server/test/util/try-with-backoff.test.ts b/packages/rtm-server/test/util/try-with-backoff.test.ts index b96d210c7..3e08c64a9 100644 --- a/packages/rtm-server/test/util/try-with-backoff.test.ts +++ b/packages/rtm-server/test/util/try-with-backoff.test.ts @@ -56,5 +56,41 @@ test('throw if maximum attempts (5) reached', async (t) => { t.is(callCount, 5); }); -// TODO allow to be cancelled +test('cancel', async (t) => { + let callCount = 0; + + const fn = () => { + callCount++; + throw new Error('test'); + }; + + const p = tryWithBackoff(fn); + p.cancel(); + + return p.then(() => { + // Cancelling won't interrupt the first callback, but it will stop it being called again + t.is(callCount, 1); + t.pass(); + }); +}); + +test('cancel nested promise', async (t) => { + let callCount = 0; + + const fn = () => { + callCount++; + if (callCount > 1) { + p.cancel(); + } + throw new Error('test'); + }; + + const p = tryWithBackoff(fn); + + return p.then(() => { + t.is(callCount, 2); + t.pass(); + }); +}); + // TODO test increasing backoffs diff --git a/packages/rtm-server/test/worker.test.ts b/packages/rtm-server/test/worker.test.ts index 89ec061ce..87c5cb0d2 100644 --- a/packages/rtm-server/test/worker.test.ts +++ b/packages/rtm-server/test/worker.test.ts @@ -18,57 +18,6 @@ import createMockRTM from '../src/mock/runtime-manager'; // This is a fake/mock websocket used by mocks -const mockChannel = (callbacks) => { - return { - push: (event: string, payload: any) => { - // if a callback was registered, trigger it - // otherwise do nothing - - let result; - if (callbacks[event]) { - result = callbacks[event](payload); - } - - return { - receive: (status, callback) => { - // TODO maybe do this asynchronously? - callback(result); - }, - }; - }, - }; -}; - -test('mock channel: should invoke handler with payload', (t) => { - return new Promise((done) => { - const channel = mockChannel({ - ping: (evt) => { - t.is(evt, 'abc'); - t.pass(); - done(); - }, - }); - - channel.push('ping', 'abc'); - }); -}); - -test('mock channel: invoke the ok handler with the callback result', (t) => { - return new Promise((done) => { - const channel = mockChannel({ - ping: () => { - return 'pong!'; - }, - }); - - channel.push('ping', 'abc').receive('ok', (evt) => { - t.is(evt, 'pong!'); - t.pass(); - done(); - }); - }); -}); - // TODO throw in the handler to get an error? test('prepareAttempt should get the attempt body', async (t) => { From 1ca68ede5ab3dedcc69fa7510108de6731c6cb37 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 22 Sep 2023 16:40:31 +0100 Subject: [PATCH 077/232] rtm: big refactor --- packages/rtm-server/src/api.ts | 6 +- .../src/{worker.ts => api/execute.ts} | 8 +- packages/rtm-server/src/api/workloop.ts | 35 ++++ packages/rtm-server/src/server.ts | 164 ++++-------------- packages/rtm-server/src/types.d.ts | 4 + .../rtm-server/src/util/try-with-backoff.ts | 6 +- packages/rtm-server/src/work-loop.ts | 51 ------ .../{worker.test.ts => api/execute.test.ts} | 12 +- packages/rtm-server/test/api/workloop.test.ts | 75 ++++++++ packages/rtm-server/test/events.test.ts | 5 +- packages/rtm-server/test/server.test.ts | 64 ------- 11 files changed, 171 insertions(+), 259 deletions(-) rename packages/rtm-server/src/{worker.ts => api/execute.ts} (96%) create mode 100644 packages/rtm-server/src/api/workloop.ts delete mode 100644 packages/rtm-server/src/work-loop.ts rename packages/rtm-server/test/{worker.test.ts => api/execute.test.ts} (96%) create mode 100644 packages/rtm-server/test/api/workloop.test.ts diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api.ts index 316adc1a3..36dace3ca 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api.ts @@ -12,14 +12,18 @@ import workflow from './middleware/workflow'; // Should have diagnostic and reporting APIs // maybe even a simple frontend? -const createAPI = (logger: Logger, execute: any) => { +const createAPI = (app: any, logger: Logger) => { const router = new Router(); router.get('/healthcheck', healthcheck); // Dev API to run a workflow + // This is totally wrong now router.post('/workflow', workflow(execute, logger)); + app.use(router.routes()); + app.use(router.allowedMethods()); + return router; }; diff --git a/packages/rtm-server/src/worker.ts b/packages/rtm-server/src/api/execute.ts similarity index 96% rename from packages/rtm-server/src/worker.ts rename to packages/rtm-server/src/api/execute.ts index d0a916f46..5d82eef57 100644 --- a/packages/rtm-server/src/worker.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -7,12 +7,12 @@ import crypto from 'node:crypto'; import phx from 'phoenix-channels'; import { JSONLog } from '@openfn/logger'; -import convertAttempt from './util/convert-attempt'; +import convertAttempt from '../util/convert-attempt'; // this managers the worker //i would like functions to be testable, and I'd like the logic to be readable -import { ATTEMPT_LOG, GET_ATTEMPT, RUN_COMPLETE, RUN_START } from './events'; -import { Attempt } from './types'; +import { ATTEMPT_LOG, GET_ATTEMPT, RUN_COMPLETE, RUN_START } from '../events'; +import { Attempt } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; export type AttemptState = { @@ -24,6 +24,7 @@ export type AttemptState = { type Channel = typeof phx.Channel; // TODO move to util + const getWithReply = (channel: Channel, event: string, payload?: any) => new Promise((resolve) => { channel.push(event, payload).receive('ok', (evt: any) => { @@ -32,6 +33,7 @@ const getWithReply = (channel: Channel, event: string, payload?: any) => // TODO handle errors amd timeouts too }); +// TODO maybe move all event handlers into api/events/* export function onJobStart( channel: Channel, state: AttemptState, diff --git a/packages/rtm-server/src/api/workloop.ts b/packages/rtm-server/src/api/workloop.ts new file mode 100644 index 000000000..8ba331b4f --- /dev/null +++ b/packages/rtm-server/src/api/workloop.ts @@ -0,0 +1,35 @@ +import { CLAIM } from '../events'; +import { CancelablePromise } from '../types'; +import tryWithBackoff from '../util/try-with-backoff'; + +// TODO this needs to return some kind of cancel function +const startWorkloop = (channel, execute, delay = 100) => { + let promise: CancelablePromise; + let cancelled = false; + + const request = () => { + channel.push(CLAIM).receive('ok', (attempts) => { + if (!attempts.length) { + // throw to backoff and try again + throw new Error('backoff'); + } + attempts.forEach((attempt) => { + execute(attempt); + }); + }); + }; + + const workLoop = () => { + if (!cancelled) { + promise = tryWithBackoff(request, { timeout: delay }); + } + }; + workLoop(); + + return () => { + cancelled = true; + promise.cancel(); + }; +}; + +export default startWorkloop; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 38d2bf030..246130f62 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -1,67 +1,13 @@ -/** - * server needs to - * - * - create a runtime manager - * - know how to speak to a lightning endpoint to fetch workflows - * Is this just a string url? - * - */ - import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import phx from 'phoenix-channels'; import { createMockLogger, Logger } from '@openfn/logger'; -import createAPI from './api'; -// import startWorkLoop from './work-loop'; -import { tryWithBackoff } from './util'; +import createRestAPI from './api-rest'; import convertAttempt from './util/convert-attempt'; -import { Attempt } from './types'; -import { CLAIM } from './events'; - -const postResult = async ( - rtmId: string, - lightningUrl: string, - attemptId: string, - state: any -) => { - if (lightningUrl) { - await fetch(`${lightningUrl}/api/1/attempts/complete/${attemptId}`, { - method: 'POST', - body: JSON.stringify({ - rtm_id: rtmId, - state: state, - }), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); - } - // TODO what if result is not 200? - // Backoff and try again? -}; - -// Send a batch of logs -const postLog = async ( - rtmId: string, - lightningUrl: string, - attemptId: string, - messages: any[] -) => { - await fetch(`${lightningUrl}/api/1/attempts/log/${attemptId}`, { - method: 'POST', - body: JSON.stringify({ - rtm_id: rtmId, - logs: messages, - }), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); -}; +import startWorkloop from './api/workloop'; +import { execute, prepareAttempt } from './api/execute'; type ServerOptions = { backoff?: number; @@ -72,12 +18,6 @@ type ServerOptions = { logger?: Logger; }; -// this is the websocket API -// basically a router -const createAPI = (ws) => { - // register events against the socket -}; - // This will open up a websocket channel to lightning // TODO auth export const connectToLightning = ( @@ -98,48 +38,12 @@ export const connectToLightning = ( }); }; -// TODO this needs to return some kind of cancel function -export const startWorkloop = (channel, execute, delay = 100) => { - let promise; - let cancelled = false; - - const request = () => { - channel.push(CLAIM).receive('ok', (attempts) => { - if (!attempts.length) { - // throw to backoff and try again - throw new Error('backoff'); - } - attempts.forEach((attempt) => { - execute(attempt); - }); - }); - }; - - const workLoop = () => { - if (!cancelled) { - promise = tryWithBackoff(request, { timeout: delay }); - // promise.then(workLoop).catch(() => { - // // this means the backoff expired - // // which right now it won't ever do - // // but what's the plan? - // // log and try again I guess? - // workLoop(); - // }); - } - }; - workLoop(); - - return () => { - cancelled = true; - promise.cancel(); - }; -}; - function createServer(rtm: any, options: ServerOptions = {}) { const logger = options.logger || createMockLogger(); const port = options.port || 1234; - logger.info('Starting server'); + logger.debug('Starting server'); + const app = new Koa(); app.use(bodyParser()); @@ -149,54 +53,56 @@ function createServer(rtm: any, options: ServerOptions = {}) { }) ); - const execute = (attempt: Attempt) => { - const plan = convertAttempt(attempt); - rtm.execute(plan); - }; - - // TODO actually it's a bit silly to pass everything through, why not just declare the route here? - // Or maybe I need a central controller/state object - const apiRouter = createAPI(logger, execute); - app.use(apiRouter.routes()); - app.use(apiRouter.allowedMethods()); + createRestAPI(app, logger); app.listen(port); - logger.success('Listening on', port); + logger.success('RTM server listening on', port); (app as any).destroy = () => { // TODO close the work loop logger.info('Closing server'); }; + const handleAttempt = async (channel, attempt) => { + const plan = await prepareAttempt(attempt); + execute(channel, rtm, plan); + }; + if (options.lightning) { logger.log('Starting work loop at', options.lightning); - connectToLightning(options.lightning, rtm.id); - //startWorkLoop(options.lightning, rtm.id, execute); + connectToLightning(options.lightning, rtm.id).then((channel) => { + // TODO maybe pull this logic out so we can test it? + startWorkloop(channel, (attempt) => { + handleAttempt(channel, attempt); + }); + + // debug API to run a workflow + // Used in unit tests + // Only loads in dev mode? + // @ts-ignore + app.execute = (attempt) => { + handleAttempt(channel, attempt); + }; + }); } else { logger.warn('No lightning URL provided'); } - rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { - logger.log(`workflow complete: `, id); - logger.log(state); - postResult(rtm.id, options.lightning!, id, state); - }); + // rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { + // logger.log(`workflow complete: `, id); + // logger.log(state); + // postResult(rtm.id, options.lightning!, id, state); + // }); - rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { - logger.log(`${id}: `, ...messages); - postLog(rtm.id, options.lightning!, id, messages); - }); + // rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { + // logger.log(`${id}: `, ...messages); + // postLog(rtm.id, options.lightning!, id, messages); + // }); // TMP doing this for tests but maybe its better done externally app.on = (...args) => rtm.on(...args); app.once = (...args) => rtm.once(...args); - // debug API to run a workflow - // Used in unit tests - // Only loads in dev mode? - // @ts-ignore - app.execute = execute; - return app; } diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index 476d7f1cb..c36cfdec3 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -47,6 +47,10 @@ export type Attempt = { worker?: string; }; +export type CancelablePromise = Promise & { + cancel: () => void; +}; + // type RuntimeExecutionPlanID = string; // type JobEdge = { diff --git a/packages/rtm-server/src/util/try-with-backoff.ts b/packages/rtm-server/src/util/try-with-backoff.ts index 718484573..6caa80eca 100644 --- a/packages/rtm-server/src/util/try-with-backoff.ts +++ b/packages/rtm-server/src/util/try-with-backoff.ts @@ -1,3 +1,5 @@ +import { CancelablePromise } from '../types'; + type Options = { attempts?: number; maxAttempts?: number; @@ -6,10 +8,6 @@ type Options = { isCancelled?: () => boolean; }; -type CancelablePromise = Promise & { - cancel: () => void; -}; - const MAX_BACKOFF = 1000 * 60; // This function will try and call its first argument every {opts.timeout|100}ms diff --git a/packages/rtm-server/src/work-loop.ts b/packages/rtm-server/src/work-loop.ts deleted file mode 100644 index e5f44dd5d..000000000 --- a/packages/rtm-server/src/work-loop.ts +++ /dev/null @@ -1,51 +0,0 @@ -import tryWithBackoff from './util/try-with-backoff'; -import { Attempt } from './types'; - -// TODO how does this report errors, like if Lightning is down? -// Or auth is bad? -export default ( - lightningUrl: string, - rtmId: string, - execute: (attempt: Attempt) => void -) => { - const fetchWork = async () => { - // TODO this needs to use the socket now - // use getWithReply to claim the attempt - // then call execute just with the id - - // // TODO what if this retuns like a 500? Server down? - // const result = await fetch(`${lightningUrl}/api/1/attempts/next`, { - // method: 'POST', - // body: JSON.stringify({ rtm_id: rtmId }), - // headers: { - // Accept: 'application/json', - // 'Content-Type': 'application/json', - // }, - // }); - // if (result.body) { - // const workflows = await result.json(); - // if (workflows.length) { - // workflows.forEach(execute); - // return true; - // } - // } - // throw to backoff and try again - throw new Error('backoff'); - }; - - const workLoop = () => { - tryWithBackoff(fetchWork) - .then(workLoop) - .catch(() => { - // this means the backoff expired - // which right now it won't ever do - // but what's the plan? - // log and try again I guess? - workLoop(); - }); - }; - - return workLoop(); - // maybe we can return an api like - // { start, pause, on('error') } -}; diff --git a/packages/rtm-server/test/worker.test.ts b/packages/rtm-server/test/api/execute.test.ts similarity index 96% rename from packages/rtm-server/test/worker.test.ts rename to packages/rtm-server/test/api/execute.test.ts index 87c5cb0d2..d273ada99 100644 --- a/packages/rtm-server/test/worker.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -1,20 +1,22 @@ import test from 'ava'; +import { JSONLog } from '@openfn/logger'; + import { GET_ATTEMPT, RUN_START, RUN_COMPLETE, ATTEMPT_LOG, -} from '../src/events'; +} from '../../src/events'; import { prepareAttempt, onJobStart, onJobComplete, onJobLog, execute, -} from '../src/worker'; -import { attempts } from './mock/data'; -import { JSONLog } from '@openfn/logger'; -import createMockRTM from '../src/mock/runtime-manager'; +} from '../../src/api/execute'; +import createMockRTM from '../../src/mock/runtime-manager'; +import { attempts } from '../mock/data'; +import { mockChannel } from '../util'; // This is a fake/mock websocket used by mocks diff --git a/packages/rtm-server/test/api/workloop.test.ts b/packages/rtm-server/test/api/workloop.test.ts new file mode 100644 index 000000000..d8a88b064 --- /dev/null +++ b/packages/rtm-server/test/api/workloop.test.ts @@ -0,0 +1,75 @@ +import test from 'ava'; +import { mockChannel, sleep } from '../util'; + +import startWorkloop from '../../src/api/workloop'; +import { CLAIM } from '../../src/events'; + +let cancel; + +test.afterEach(() => { + cancel?.(); // cancel any workloops +}); + +test('workloop can be cancelled', async (t) => { + let count = 0; + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + count++; + cancel(); + }, + }); + + cancel = startWorkloop(channel, () => {}, 1); + + await sleep(100); + // A quirk of how cancel works is that the loop will be called a few times + t.assert(count <= 5); +}); + +test('workloop sends the attempts:claim event', (t) => { + return new Promise((done) => { + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + t.pass(); + done(); + }, + }); + cancel = startWorkloop(channel, () => {}); + }); +}); + +test('workloop sends the attempts:claim event several times ', (t) => { + return new Promise((done) => { + let cancel; + let count = 0; + const channel = mockChannel({ + [CLAIM]: () => { + count++; + if (count === 5) { + t.pass(); + done(); + } + }, + }); + cancel = startWorkloop(channel, () => {}); + }); +}); + +test('workloop calls execute if attempts:claim returns attempts', (t) => { + return new Promise((done) => { + let cancel; + const channel = mockChannel({ + [CLAIM]: () => { + return [{ id: 'a' }]; + }, + }); + + cancel = startWorkloop(channel, (attempt) => { + t.deepEqual(attempt, { id: 'a' }); + t.pass(); + done(); + }); + }); +}); diff --git a/packages/rtm-server/test/events.test.ts b/packages/rtm-server/test/events.test.ts index 5678e7dc6..45ef2edc6 100644 --- a/packages/rtm-server/test/events.test.ts +++ b/packages/rtm-server/test/events.test.ts @@ -1,3 +1,4 @@ +// TODO This can all be removed /** * Unit tests on events published by the rtm-server * No lightning involved here @@ -16,7 +17,7 @@ test.before(() => { server = createRTMServer(rtm, { port: 2626 }); }); -test.serial( +test.serial.skip( 'trigger a workflow-start event when execution starts', async (t) => { server.execute({ @@ -34,7 +35,7 @@ test.serial( } ); -test.serial.only( +test.serial.skip( 'trigger a workflow-complete event when execution completes', async (t) => { server.execute({ diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index 938c3ee26..a5074c03c 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -40,70 +40,6 @@ test('connects to lightning', async (t) => { // TODO connections to hte same socket.channel should share listners, so I think I can test the channel }); -test('workloop can be cancelled', async (t) => { - let count = 0; - let cancel; - const channel = mockChannel({ - [CLAIM]: () => { - count++; - cancel(); - }, - }); - - cancel = startWorkloop(channel, () => {}, 1); - - await sleep(100); - // A quirk of how cancel works is that the loop will be called a few times - t.assert(count <= 5); -}); - -test('workloop sends the attempts:claim event', (t) => { - return new Promise((done) => { - let cancel; - const channel = mockChannel({ - [CLAIM]: () => { - t.pass(); - done(); - }, - }); - cancel = startWorkloop(channel, () => {}); - }); -}); - -test('workloop sends the attempts:claim event several times ', (t) => { - return new Promise((done) => { - let cancel; - let count = 0; - const channel = mockChannel({ - [CLAIM]: () => { - count++; - if (count === 5) { - t.pass(); - done(); - } - }, - }); - cancel = startWorkloop(channel, () => {}); - }); -}); - -test('workloop calls execute if attempts:claim returns attempts', (t) => { - return new Promise((done) => { - let cancel; - const channel = mockChannel({ - [CLAIM]: () => { - return [{ id: 'a' }]; - }, - }); - - cancel = startWorkloop(channel, (attempt) => { - t.deepEqual(attempt, { id: 'a' }); - t.pass(); - done(); - }); - }); -}); - // test('connects to websocket', (t) => { // let didSayHello; From a9d31f6c71fe08a19ba6aac07dd69b346884b1c7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 25 Sep 2023 17:00:09 +0100 Subject: [PATCH 078/232] rtm: hookup lightning and rtm server so that we can process a complete event big duplication of events though --- packages/rtm-server/package.json | 2 +- .../rtm-server/src/{api.ts => api-rest.ts} | 2 +- packages/rtm-server/src/api/execute.ts | 32 +++++++++++++-- packages/rtm-server/src/api/workloop.ts | 18 ++++---- .../rtm-server/src/mock/lightning/api-dev.ts | 4 ++ .../src/mock/lightning/api-sockets.ts | 36 +++++++++++++++- .../src/mock/lightning/socket-server.ts | 1 + .../rtm-server/src/mock/runtime-manager.ts | 6 +++ packages/rtm-server/src/server.ts | 41 +++++++++++++------ packages/rtm-server/src/start.ts | 11 +++-- packages/rtm-server/test/api/workloop.test.ts | 2 +- packages/rtm-server/test/util.ts | 3 +- 12 files changed, 125 insertions(+), 33 deletions(-) rename packages/rtm-server/src/{api.ts => api-rest.ts} (92%) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 46b3215a8..e0818187b 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -10,7 +10,7 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", - "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm src/start.ts", + "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm --transpile-only src/start.ts", "start:lightning": "ts-node-esm --transpile-only src/mock/lightning/start.ts", "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'" }, diff --git a/packages/rtm-server/src/api.ts b/packages/rtm-server/src/api-rest.ts similarity index 92% rename from packages/rtm-server/src/api.ts rename to packages/rtm-server/src/api-rest.ts index 36dace3ca..febd9059f 100644 --- a/packages/rtm-server/src/api.ts +++ b/packages/rtm-server/src/api-rest.ts @@ -19,7 +19,7 @@ const createAPI = (app: any, logger: Logger) => { // Dev API to run a workflow // This is totally wrong now - router.post('/workflow', workflow(execute, logger)); + // router.post('/workflow', workflow(execute, logger)); app.use(router.routes()); app.use(router.allowedMethods()); diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 5d82eef57..d0ca77c97 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -11,7 +11,13 @@ import convertAttempt from '../util/convert-attempt'; // this managers the worker //i would like functions to be testable, and I'd like the logic to be readable -import { ATTEMPT_LOG, GET_ATTEMPT, RUN_COMPLETE, RUN_START } from '../events'; +import { + ATTEMPT_COMPLETE, + ATTEMPT_LOG, + GET_ATTEMPT, + RUN_COMPLETE, + RUN_START, +} from '../events'; import { Attempt } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; @@ -70,6 +76,16 @@ export function onJobComplete( delete state.activeJob; } +export function onWorkflowComplete(channel: Channel, state, evt) { + channel.push(ATTEMPT_COMPLETE, { + // no point in publishing the workflow id + // TODO include final data + dataclip: evt.state, + }); + + // TODO should I mark the state? Is there any point? +} + export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { // we basically just forward the log to lightning // but we also need to attach the log id @@ -86,6 +102,7 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { export async function prepareAttempt(channel: Channel) { // first we get the attempt body through the socket const attemptBody = (await getWithReply(channel, GET_ATTEMPT)) as Attempt; + console.log(attemptBody); // then we generate the execution plan const plan = convertAttempt(attemptBody); @@ -108,6 +125,8 @@ async function loadCredential(ws, attemptId, stateId) {} // TODO actually this now is a Workflow or Execution plan // It's not an attempt anymore export function execute(channel: Channel, rtm, plan: ExecutionPlan) { + // TODO add proper logger (maybe channel, rtm and logger comprise a context object) + console.log('execute', plan); // tracking state for this attempt const state: AttemptState = { //attempt, // keep this on the state so that anyone can access it @@ -119,10 +138,15 @@ export function execute(channel: Channel, rtm, plan: ExecutionPlan) { // this is super declarative // TODO is there any danger of events coming through out of order? // what if onJoblog takes 1 second to finish and before the runId is set, onJobLog comes through? + + // TODO + // const context = { channel, state } + rtm.listen(plan.id, { - 'job-start': (evt) => onJobStart(plan, state, evt), - 'job-complete': (evt) => onJobComplete(plan, state, evt), - 'job-log': (evt) => onJobLog(plan, state, evt), + 'job-start': (evt) => onJobStart(channel, state, evt), + 'job-complete': (evt) => onJobComplete(channel, state, evt), + 'job-log': (evt) => onJobLog(channel, state, evt), + 'workflow-complete': (evt) => onWorkflowComplete(channel, state, evt), }); rtm.execute(plan); diff --git a/packages/rtm-server/src/api/workloop.ts b/packages/rtm-server/src/api/workloop.ts index 8ba331b4f..2928dd0e8 100644 --- a/packages/rtm-server/src/api/workloop.ts +++ b/packages/rtm-server/src/api/workloop.ts @@ -8,13 +8,17 @@ const startWorkloop = (channel, execute, delay = 100) => { let cancelled = false; const request = () => { - channel.push(CLAIM).receive('ok', (attempts) => { - if (!attempts.length) { - // throw to backoff and try again - throw new Error('backoff'); - } - attempts.forEach((attempt) => { - execute(attempt); + return new Promise((resolve, reject) => { + channel.push(CLAIM).receive('ok', (attempts) => { + if (!attempts?.length) { + // throw to backoff and try again + return reject(new Error('backoff')); + } + + attempts.forEach((attempt) => { + execute(attempt); + resolve(); + }); }); }); }; diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index 7c58b7cd3..f87dbd32d 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -42,6 +42,9 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.enqueueAttempt = (attempt: Attempt) => { state.attempts[attempt.id] = attempt; state.results[attempt.id] = {}; + state.pending[attempt.id] = { + status: 'queued', + }; state.queue.push(attempt.id); }; @@ -100,6 +103,7 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { const attempt = ctx.request.body; logger.info('Adding new attempt to queue:', attempt.id); + logger.debug(attempt); if (!attempt.id) { attempt.id = crypto.randomUUID(); diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index 650660924..1bba6a60f 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -6,7 +6,13 @@ import type { ServerState } from './server'; import { extractAttemptId } from './util'; import createPheonixMockSocketServer from './socket-server'; -import { CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP } from '../../events'; +import { + ATTEMPT_COMPLETE, + CLAIM, + GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, +} from '../../events'; // this new API is websocket based // Events map to handlers @@ -60,7 +66,7 @@ const createSocketAPI = ( payload.response.push(next); count -= 1; - startAttempt(next.id); + startAttempt(next); } if (payload.response.length) { logger?.info(`Claiming ${payload.response.length} attempts`); @@ -114,6 +120,30 @@ const createSocketAPI = ( }); }; + // TODO why is this firing a million times? + const handleAttemptComplete = (state, ws, evt) => { + const { id, ref, topic, dataclip } = evt; + + // TODO use proper logger + console.log('Completed attempted ', id); + console.log(dataclip); + + // TODO what does the mock do here? + // well, we should acknowlege + // Should we error if there's no dataclip? + // but who does that help? + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + // response: { + // dataclip, + // }, + }, + }); + }; + wss.registerEvents('attempts:queue', { [CLAIM]: (ws, event) => pullClaim(state, ws, event), }); @@ -132,6 +162,8 @@ const createSocketAPI = ( [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), + [ATTEMPT_COMPLETE]: (ws, event) => + handleAttemptComplete(state, ws, { id: attemptId, ...event }), }); }; diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 15af240be..3f85cbcad 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -51,6 +51,7 @@ function createServer({ // TODO is this logic in the right place? if (topic.startsWith(ATTEMPT_PREFIX)) { + console.log(state); const attemptId = extractAttemptId(topic); if (!state.pending[attemptId]) { status = 'error'; diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index 4b26b09ba..ffc5cb617 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -71,6 +71,7 @@ function createMock( const listeners: Record = {}; const dispatch = (type: RTMEvent, args?: any) => { + console.log(' > ', type, args); if (args.workflowId) { listeners[args.workflowId]?.[type]?.(args); } @@ -136,10 +137,13 @@ function createMock( // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = (xplan: ExecutionPlan) => { + console.log('[mockrtm] execute'); + console.log(xplan); const { id, jobs } = xplan; const workflowId = id; activeWorkflows[id!] = true; setTimeout(() => { + console.log('[mockrtm] start'); dispatch('workflow-start', { workflowId }); setTimeout(async () => { let state = {}; @@ -149,6 +153,8 @@ function createMock( } setTimeout(() => { delete activeWorkflows[id!]; + console.log('[mockrtm] complete'); + console.log(state); dispatch('workflow-complete', { workflowId, state }); // TODO on workflow complete we should maybe tidy the listeners? // Doesn't really matter in the mock though diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 246130f62..99a95ad8a 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -27,14 +27,26 @@ export const connectToLightning = ( ) => { return new Promise((done) => { let socket = new Socket(endpoint /*,{params: {userToken: "123"}}*/); - socket.connect(); - - // join the queue channel - const channel = socket.channel('attempts:queue'); - channel.join().receive('ok', () => { - done(channel); + // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) + // Do we infinitely try to reconnect? + // Consider what happens when the connection drops + // Unit tests on all of these behaviours! + socket.onOpen(() => { + // join the queue channel + const channel = socket.channel('attempts:queue'); + + channel + .join() + .receive('ok', () => { + done([socket, channel]); + }) + .receive('error', (e) => { + console.log('ERROR', err); + }); }); + + socket.connect(); }); }; @@ -63,17 +75,22 @@ function createServer(rtm: any, options: ServerOptions = {}) { logger.info('Closing server'); }; - const handleAttempt = async (channel, attempt) => { - const plan = await prepareAttempt(attempt); - execute(channel, rtm, plan); + // TODO this needs loads more unit testing - deffo need to pull it into its own funciton + const handleAttempt = async (socket, attempt) => { + const channel = socket.channel(`attempt:${attempt}`); + channel.join().receive('ok', async () => { + const plan = await prepareAttempt(channel); + console.log(plan); + execute(channel, rtm, plan); + }); }; if (options.lightning) { logger.log('Starting work loop at', options.lightning); - connectToLightning(options.lightning, rtm.id).then((channel) => { + connectToLightning(options.lightning, rtm.id).then(([socket, channel]) => { // TODO maybe pull this logic out so we can test it? startWorkloop(channel, (attempt) => { - handleAttempt(channel, attempt); + handleAttempt(socket, attempt); }); // debug API to run a workflow @@ -81,7 +98,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { // Only loads in dev mode? // @ts-ignore app.execute = (attempt) => { - handleAttempt(channel, attempt); + handleAttempt(socket, attempt); }; }); } else { diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index d8a1e5aed..3d72ae27c 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -4,6 +4,7 @@ import { hideBin } from 'yargs/helpers'; import createLogger from '@openfn/logger'; import createRTM from '@openfn/runtime-manager'; +import createMockRTM from './mock/runtime-manager'; import createRTMServer from './server'; type Args = { @@ -35,7 +36,7 @@ const args = yargs(hideBin(process.argv)) .parse() as Args; if (args.lightning === 'mock') { - args.lightning = 'http://localhost:8888'; + args.lightning = 'ws://localhost:8888/api'; } // TODO the rtm needs to take callbacks to load credential, and load state @@ -44,8 +45,12 @@ if (args.lightning === 'mock') { // Or the server calls a setCalbacks({ credential, state }) function on the RTM // Each of these takes the attemptId as the firsdt argument // credential and state will lookup the right channel -const rtm = createRTM('rtm', { repoDir: args.repoDir }); -logger.debug('RTM created'); +// const rtm = createRTM('rtm', { repoDir: args.repoDir }); +// logger.debug('RTM created'); + +// use the mock rtm for now +const rtm = createMockRTM('rtm'); +logger.debug('Mock RTM created'); createRTMServer(rtm, { port: args.port, diff --git a/packages/rtm-server/test/api/workloop.test.ts b/packages/rtm-server/test/api/workloop.test.ts index d8a88b064..57fef7bb7 100644 --- a/packages/rtm-server/test/api/workloop.test.ts +++ b/packages/rtm-server/test/api/workloop.test.ts @@ -40,7 +40,7 @@ test('workloop sends the attempts:claim event', (t) => { }); }); -test('workloop sends the attempts:claim event several times ', (t) => { +test.only('workloop sends the attempts:claim event several times ', (t) => { return new Promise((done) => { let cancel; let count = 0; diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index 04f5cc806..51fc8dddd 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -46,8 +46,7 @@ export const mockChannel = (callbacks = {}) => { return { receive: (status, callback) => { - // TODO maybe do this asynchronously? - callback(result); + setTimeout(() => callback(result), 1); }, }; }, From 33d5b9c13790e390bed145013efc21d516235a0a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 10:22:18 +0100 Subject: [PATCH 079/232] rtm: tidying up in the lightning mock --- packages/rtm-server/package.json | 1 + .../src/mock/lightning/api-sockets.ts | 115 +++++++++--------- .../rtm-server/src/mock/lightning/server.ts | 25 ++-- .../src/mock/lightning/socket-server.ts | 81 ++++++++---- .../rtm-server/test/mock/lightning.test.ts | 19 +++ pnpm-lock.yaml | 15 ++- 6 files changed, 163 insertions(+), 93 deletions(-) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index e0818187b..5c3903286 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -22,6 +22,7 @@ "@openfn/runtime": "workspace:*", "@openfn/runtime-manager": "workspace:*", "@types/koa-logger": "^3.1.2", + "@types/ws": "^8.5.6", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", "koa-logger": "^3.2.1", diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index 1bba6a60f..da2d61fa5 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -5,7 +5,10 @@ import type { ServerState } from './server'; import { extractAttemptId } from './util'; -import createPheonixMockSocketServer from './socket-server'; +import createPheonixMockSocketServer, { + DevSocket, + PhoenixEvent, +} from './socket-server'; import { ATTEMPT_COMPLETE, CLAIM, @@ -14,6 +17,8 @@ import { GET_DATACLIP, } from '../../events'; +import type { Server } from 'http'; + // this new API is websocket based // Events map to handlers // can I even implement this in JS? Not with pheonix anyway. hmm. @@ -22,11 +27,10 @@ import { const createSocketAPI = ( state: ServerState, path: string, - httpServer, + httpServer: Server, logger?: Logger ) => { // set up a websocket server to listen to connections - // console.log('path', path); const server = new WebSocketServer({ server: httpServer, @@ -41,15 +45,37 @@ const createSocketAPI = ( logger: logger && createLogger('PHX', { level: 'debug' }), }); - // TODO - // 1) Need to improve the abstraction of these, make messages easier to send - // 2) Also need to look at closures - I'd like a declarative central API - // the need to call startAttempt makes things a bit harder + wss.registerEvents('attempts:queue', { + [CLAIM]: (ws, event) => pullClaim(state, ws, event), + }); + + const startAttempt = (attemptId: string) => { + // mark the attempt as started on the server + state.pending[attemptId] = { + status: 'started', + }; + + // TODO do all these need extra auth, or is auth granted + // implicitly by channel membership? + // Right now the socket gets access to all server state + // But this is just a mock - Lightning can impose more restrictions if it wishes + wss.registerEvents(`attempt:${attemptId}`, { + [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), + [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), + [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), + [ATTEMPT_COMPLETE]: (ws, event) => + handleAttemptComplete(state, ws, event, attemptId), + }); + }; + + return { + startAttempt, + }; // pull claim will try and pull a claim off the queue, // and reply with the response // the reply ensures that only the calling worker will get the attempt - const pullClaim = (state, ws, evt) => { + function pullClaim(state: ServerState, ws, evt) { const { ref, topic } = evt; const { queue } = state; let count = 1; @@ -75,9 +101,9 @@ const createSocketAPI = ( } ws.reply({ ref, topic, payload }); - }; + } - const getAttempt = (state, ws, evt) => { + function getAttempt(state: ServerState, ws, evt) { const { ref, topic } = evt; const attemptId = extractAttemptId(topic); const attempt = state.attempts[attemptId]; @@ -90,12 +116,12 @@ const createSocketAPI = ( response: attempt, }, }); - }; + } - const getCredential = (state, ws, evt) => { + function getCredential(state: ServerState, ws, evt) { const { ref, topic, payload, event } = evt; const response = state.credentials[payload.id]; - // console.log(topic, event, response); + console.log(topic, event, response); ws.reply({ ref, topic, @@ -104,12 +130,12 @@ const createSocketAPI = ( response, }, }); - }; + } - const getDataclip = (state, ws, evt) => { - const { ref, topic, payload, event } = evt; + function getDataclip(state: ServerState, ws, evt) { + const { ref, topic, payload } = evt; const response = state.dataclips[payload.id]; - console.log(response); + ws.reply({ ref, topic, @@ -118,58 +144,33 @@ const createSocketAPI = ( response, }, }); - }; + } // TODO why is this firing a million times? - const handleAttemptComplete = (state, ws, evt) => { - const { id, ref, topic, dataclip } = evt; + function handleAttemptComplete( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent, + attemptId: string + ) { + const { ref, topic, payload } = evt; + const { dataclip } = payload; // TODO use proper logger - console.log('Completed attempted ', id); - console.log(dataclip); + logger?.info('Completed attempted ', attemptId); + logger?.debug(dataclip); + + state.pending[attemptId].status = 'complete'; + state.results[attemptId] = dataclip; - // TODO what does the mock do here? - // well, we should acknowlege - // Should we error if there's no dataclip? - // but who does that help? ws.reply({ ref, topic, payload: { status: 'ok', - // response: { - // dataclip, - // }, }, }); - }; - - wss.registerEvents('attempts:queue', { - [CLAIM]: (ws, event) => pullClaim(state, ws, event), - }); - - const startAttempt = (attemptId) => { - // mark the attempt as started on the server - state.pending[attemptId] = { - status: 'started', - }; - - // TODO do all these need extra auth, or is auth granted - // implicitly by channel membership? - // Right now the socket gets access to all server state - // But this is just a mock - Lightning can impose more restrictions if it wishes - wss.registerEvents(`attempt:${attemptId}`, { - [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), - [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), - [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), - [ATTEMPT_COMPLETE]: (ws, event) => - handleAttemptComplete(state, ws, { id: attemptId, ...event }), - }); - }; - - return { - startAttempt, - }; + } }; export default createSocketAPI; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 7c8a3cfa5..79f4c30be 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -20,10 +20,25 @@ import { Attempt } from '../../types'; export const API_PREFIX = '/api/1'; export type ServerState = { + queue: AttemptId[]; + + // list of credentials by id credentials: Record; + + // list of events by id attempts: Record; - queue: Attempt[]; + + // list of dataclips by id + dataclips: Record; + + // Tracking state of known attempts + // TODO include the rtm id and token + pending: Record; + + // Track all completed attempts here results: Record; + + // event emitter for debugging and observability events: EventEmitter; }; @@ -40,17 +55,9 @@ const createLightningServer = (options: LightningOptions = {}) => { const logger = options.logger || createMockLogger(); const state = { - // list of credentials by id credentials: {}, - // list of events by id attempts: {}, - - // list of dataclips by id dataclips: {}, - - // attempts which have been started - // probaby need to track status and maybe the rtm id? - // TODO maybe Active is a better word? pending: {}, queue: [] as AttemptId[], diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 3f85cbcad..c9565243f 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -1,22 +1,50 @@ -import { WebSocketServer } from 'ws'; +/** + * This module creates a mock pheonix socket server + * It uses a standard ws server but wraps messages up in a + * structure that pheonix sockets can understand + * It also adds some dev and debug APIs, useful for unit testing + */ +import { WebSocketServer, WebSocket } from 'ws'; + import { ATTEMPT_PREFIX, extractAttemptId } from './util'; +import { ServerState } from './server'; -// mock pheonix websocket server +import type { Logger } from '@openfn/logger'; -// - route messages to rooms -// - respond ok to connections type Topic = string; -type WS = any; +// websocket with a couple of dev-friendly APIs +export type DevSocket = WebSocket & { + reply: (evt: Pick) => void; + sendJSON: ({ event, topic, ref }: PhoenixEvent) => void; +}; -type PhoenixEvent = { +export type PhoenixEvent

= { topic: Topic; event: string; - payload?: any; + payload?: P; ref?: string; }; -type EventHandler = (event: string, payload: any) => void; +type EventHandler = (ws: DevSocket, event: PhoenixEvent) => void; + +type CreateServerOptions = { + port?: number; + server: typeof WebSocketServer; + state: ServerState; + logger?: Logger; + onMessage?: (evt: PhoenixEvent) => void; +}; + +type MockSocketServer = typeof WebSocketServer & { + // Dev/debug APIs + listenToChannel: ( + topic: Topic, + fn: EventHandler + ) => { unsubscribe: () => void }; + waitForMessage: (topic: Topic, event: string) => Promise; + registerEvents: (topic: Topic, events: Record) => void; +}; function createServer({ port = 8080, @@ -24,7 +52,7 @@ function createServer({ state, logger, onMessage = () => {}, -} = {}) { +}: CreateServerOptions) { logger?.info('pheonix mock websocket server listening on', port); const channels: Record> = {}; @@ -36,7 +64,7 @@ function createServer({ const events = { // testing (TODO shouldn't this be in a specific channel?) - ping: (ws, { topic, ref }) => { + ping: (ws: DevSocket, { topic, ref }: PhoenixEvent) => { ws.sendJSON({ topic, ref, @@ -45,13 +73,12 @@ function createServer({ }); }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket - phx_join: (ws, { event, topic, ref }) => { + phx_join: (ws: DevSocket, { topic, ref }: PhoenixEvent) => { let status = 'ok'; let response = 'ok'; // TODO is this logic in the right place? if (topic.startsWith(ATTEMPT_PREFIX)) { - console.log(state); const attemptId = extractAttemptId(topic); if (!state.pending[attemptId]) { status = 'error'; @@ -66,14 +93,18 @@ function createServer({ }, }; - wsServer.on('connection', function (ws: WS, req) { - // TODO need to be logging here really - logger?.info('new connection'); + wsServer.on('connection', function (ws: DevSocket, _req: any) { + logger?.info('new client connected'); - ws.reply = ({ ref, topic, payload }: PhoenixEvent) => { + ws.reply = ({ + ref, + topic, + payload, + }: Pick) => { logger?.debug( `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) ); + // console.log('reply', topic, ref, payload); ws.send( JSON.stringify({ event: `chan_reply_${ref}`, @@ -123,8 +154,10 @@ function createServer({ }); }); + const mockServer = wsServer as MockSocketServer; + // debugAPI - wsServer.listenToChannel = (topic: Topic, fn: EventHandler) => { + mockServer.listenToChannel = (topic: Topic, fn: EventHandler) => { if (!channels[topic]) { channels[topic] = new Set(); } @@ -138,25 +171,25 @@ function createServer({ }; }; - wsServer.waitForMessage = (topic: Topic, event: string) => { - return new Promise((resolve) => { - const listener = wsServer.listenToChannel(topic, (ws, e) => { + mockServer.waitForMessage = (topic: Topic, event: string) => { + return new Promise((resolve) => { + const listener = mockServer.listenToChannel(topic, (_ws, e) => { if (e.event === event) { listener.unsubscribe(); - resolve(event); + resolve(e); } }); }); }; // TODO how do we unsubscribe? - wsServer.registerEvents = (topic: Topic, events) => { + mockServer.registerEvents = (topic: Topic, events) => { for (const evt in events) { - wsServer.listenToChannel(topic, events[evt]); + mockServer.listenToChannel(topic, events[evt]); } }; - return wsServer; + return mockServer as MockSocketServer; } export default createServer; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index bce4c3c6a..513e96388 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -5,6 +5,7 @@ import phx from 'phoenix-channels'; import { attempts, credentials, dataclips } from './data'; import { + ATTEMPT_COMPLETE, CLAIM, GET_ATTEMPT, GET_CREDENTIAL, @@ -236,6 +237,24 @@ test.serial('get attempt data through the attempt channel', async (t) => { }); }); +test.serial('complete an attempt through the attempt channel', async (t) => { + return new Promise(async (done) => { + const a = attempt1; + server.registerAttempt(a); + server.startAttempt(a.id); + + const channel = await join(`attempt:${a.id}`); + channel + .push(ATTEMPT_COMPLETE, { dataclip: { answer: 42 } }) + .receive('ok', () => { + const { pending, results } = server.getState(); + t.deepEqual(pending[a.id], { status: 'complete' }); + t.deepEqual(results[a.id], { answer: 42 }); + done(); + }); + }); +}); + // TODO can't work out why this is failing - there's just no data in the response test.serial.skip('get credential through the attempt channel', async (t) => { return new Promise(async (done) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2352ed1e4..67c24dcfb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -384,6 +384,9 @@ importers: '@types/koa-logger': specifier: ^3.1.2 version: 3.1.2 + '@types/ws': + specifier: ^8.5.6 + version: 8.5.6 koa: specifier: ^2.13.4 version: 2.13.4 @@ -1496,7 +1499,7 @@ packages: resolution: {integrity: sha512-DTuBFbqu4gGfajREEMrkq5jBhcnskinhr4+AnfJEk48zhVeEv3XnUKGIX98B74kxhYsIMfApGGySTn7V3b5yBA==} engines: {node: '>= 12.13.0', npm: '>= 6.12.0'} dependencies: - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@slack/types@2.8.0: @@ -1599,7 +1602,7 @@ packages: resolution: {integrity: sha512-IO+MJPVhoqz+28h1qLAcBEH2+xHMK6MTyHJc7MTnnYb6wsoLR29POVGJ7LycmVXIqyy/4/2ShP5sUwTXuOwb/w==} dependencies: '@types/minimatch': 5.1.2 - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@types/gunzip-maybe@1.4.0: @@ -1623,7 +1626,7 @@ packages: /@types/is-stream@1.1.0: resolution: {integrity: sha512-jkZatu4QVbR60mpIzjINmtS1ZF4a/FqdTUTBeQDVOQ2PYyidtwFKr0B5G6ERukKwliq+7mIXvxyppwzG5EgRYg==} dependencies: - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@types/json-diff@1.0.0: @@ -1805,6 +1808,12 @@ packages: resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} dev: false + /@types/ws@8.5.6: + resolution: {integrity: sha512-8B5EO9jLVCy+B58PLHvLDuOD8DRVMgQzq8d55SjLCOn9kqGyqOvy27exVaTio1q1nX5zLu8/6N0n2ThSxOM6tg==} + dependencies: + '@types/node': 18.15.13 + dev: false + /@types/yargs-parser@21.0.0: resolution: {integrity: sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==} dev: true From 0e63800f4a15452f2d656dbbc8c67d5e6bb79d58 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 11:25:13 +0100 Subject: [PATCH 080/232] rtm: fix duplicated messages in lightning mock, expand unit tests --- .../src/mock/lightning/api-sockets.ts | 12 ++++--- .../src/mock/lightning/socket-server.ts | 33 ++++++++++++++----- .../rtm-server/test/mock/lightning.test.ts | 24 +++++++++++--- 3 files changed, 52 insertions(+), 17 deletions(-) diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index da2d61fa5..6af6d9823 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -59,12 +59,14 @@ const createSocketAPI = ( // implicitly by channel membership? // Right now the socket gets access to all server state // But this is just a mock - Lightning can impose more restrictions if it wishes - wss.registerEvents(`attempt:${attemptId}`, { + const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), - [ATTEMPT_COMPLETE]: (ws, event) => - handleAttemptComplete(state, ws, event, attemptId), + [ATTEMPT_COMPLETE]: (ws, event) => { + handleAttemptComplete(state, ws, event, attemptId); + unsubscribe(); + }, }); }; @@ -119,9 +121,9 @@ const createSocketAPI = ( } function getCredential(state: ServerState, ws, evt) { - const { ref, topic, payload, event } = evt; + const { ref, topic, payload } = evt; const response = state.credentials[payload.id]; - console.log(topic, event, response); + // console.log(topic, event, response); ws.reply({ ref, topic, diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index c9565243f..499f4baf3 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -43,7 +43,10 @@ type MockSocketServer = typeof WebSocketServer & { fn: EventHandler ) => { unsubscribe: () => void }; waitForMessage: (topic: Topic, event: string) => Promise; - registerEvents: (topic: Topic, events: Record) => void; + registerEvents: ( + topic: Topic, + events: Record + ) => { unsubscribe: () => void }; }; function createServer({ @@ -104,7 +107,6 @@ function createServer({ logger?.debug( `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) ); - // console.log('reply', topic, ref, payload); ws.send( JSON.stringify({ event: `chan_reply_${ref}`, @@ -144,10 +146,20 @@ function createServer({ events[event](ws, { topic, payload, ref }); } else { // handle custom/user events - if (channels[topic]) { + if (channels[topic] && channels[topic].size) { channels[topic].forEach((fn) => { fn(ws, { event, topic, payload, ref }); }); + } else { + // This behaviour is just a convenience for unit tesdting + ws.reply({ + ref, + topic, + payload: { + status: 'error', + response: `There are no listeners on channel ${topic}`, + }, + }); } } } @@ -156,7 +168,8 @@ function createServer({ const mockServer = wsServer as MockSocketServer; - // debugAPI + // debug API + // TODO should this in fact be (topic, event, fn)? mockServer.listenToChannel = (topic: Topic, fn: EventHandler) => { if (!channels[topic]) { channels[topic] = new Set(); @@ -182,11 +195,15 @@ function createServer({ }); }; - // TODO how do we unsubscribe? mockServer.registerEvents = (topic: Topic, events) => { - for (const evt in events) { - mockServer.listenToChannel(topic, events[evt]); - } + // listen to all events in the channel + return mockServer.listenToChannel(topic, (ws, evt) => { + const { event } = evt; + // call the correct event handler for this event + if (events[event]) { + events[event](ws, evt); + } + }); }; return mockServer as MockSocketServer; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 513e96388..b2ab26d38 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -255,8 +255,25 @@ test.serial('complete an attempt through the attempt channel', async (t) => { }); }); -// TODO can't work out why this is failing - there's just no data in the response -test.serial.skip('get credential through the attempt channel', async (t) => { +test.serial('unusubscribe after attempt complete', async (t) => { + return new Promise(async (done) => { + const a = attempt1; + server.registerAttempt(a); + server.startAttempt(a.id); + + const channel = await join(`attempt:${a.id}`); + channel.push(ATTEMPT_COMPLETE).receive('ok', () => { + // After the complete event, the listener should unsubscribe to the channel + // The mock will send an error to any unhandled events in that channel + channel.push(ATTEMPT_COMPLETE).receive('error', () => { + t.pass(); + done(); + }); + }); + }); +}); + +test.serial('get credential through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addCredential('a', credentials['a']); @@ -269,8 +286,7 @@ test.serial.skip('get credential through the attempt channel', async (t) => { }); }); -// TODO can't work out why this is failing - there's just no data in the response -test.serial.skip('get dataclip through the attempt channel', async (t) => { +test.serial('get dataclip through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addDataclip('d', dataclips['d']); From aa92b246a6da52c1bcb516eb384aae4f3fcd2a34 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 11:39:24 +0100 Subject: [PATCH 081/232] rtm: add support for logging to the mock --- packages/rtm-server/src/events.ts | 2 +- .../src/mock/lightning/api-sockets.ts | 33 +++++++++++++++---- .../rtm-server/src/mock/lightning/server.ts | 14 ++++---- .../rtm-server/test/mock/lightning.test.ts | 29 ++++++++++++++-- 4 files changed, 61 insertions(+), 17 deletions(-) diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 904b97437..d2dff5a8c 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -17,7 +17,7 @@ export const GET_DATACLIP = 'fetch:dataclip'; export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats -export const ATTEMPT_LOG = 'attempt:complete'; // level, namespace (job,runtime,adaptor), message, time +export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time // this should not happen - this is "could not execute" rather than "complete with errors" export const ATTEMPT_ERROR = 'attempt:error'; diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index 6af6d9823..a65de70bf 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -11,6 +11,7 @@ import createPheonixMockSocketServer, { } from './socket-server'; import { ATTEMPT_COMPLETE, + ATTEMPT_LOG, CLAIM, GET_ATTEMPT, GET_CREDENTIAL, @@ -53,6 +54,7 @@ const createSocketAPI = ( // mark the attempt as started on the server state.pending[attemptId] = { status: 'started', + logs: [], }; // TODO do all these need extra auth, or is auth granted @@ -63,10 +65,14 @@ const createSocketAPI = ( [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), + [ATTEMPT_LOG]: (ws, event) => handleLog(state, ws, event), [ATTEMPT_COMPLETE]: (ws, event) => { handleAttemptComplete(state, ws, event, attemptId); unsubscribe(); }, + // TODO + // [RUN_START] + // [RUN_COMPLETE] }); }; @@ -77,7 +83,7 @@ const createSocketAPI = ( // pull claim will try and pull a claim off the queue, // and reply with the response // the reply ensures that only the calling worker will get the attempt - function pullClaim(state: ServerState, ws, evt) { + function pullClaim(state: ServerState, ws: DevSocket, evt) { const { ref, topic } = evt; const { queue } = state; let count = 1; @@ -105,7 +111,7 @@ const createSocketAPI = ( ws.reply({ ref, topic, payload }); } - function getAttempt(state: ServerState, ws, evt) { + function getAttempt(state: ServerState, ws: DevSocket, evt) { const { ref, topic } = evt; const attemptId = extractAttemptId(topic); const attempt = state.attempts[attemptId]; @@ -120,7 +126,7 @@ const createSocketAPI = ( }); } - function getCredential(state: ServerState, ws, evt) { + function getCredential(state: ServerState, ws: DevSocket, evt) { const { ref, topic, payload } = evt; const response = state.credentials[payload.id]; // console.log(topic, event, response); @@ -134,7 +140,7 @@ const createSocketAPI = ( }); } - function getDataclip(state: ServerState, ws, evt) { + function getDataclip(state: ServerState, ws: DevSocket, evt) { const { ref, topic, payload } = evt; const response = state.dataclips[payload.id]; @@ -148,7 +154,21 @@ const createSocketAPI = ( }); } - // TODO why is this firing a million times? + function handleLog(state: ServerState, ws: DevSocket, evt) { + const { ref, topic, payload } = evt; + const { attempt_id: attemptId } = payload; + + state.pending[attemptId].logs.push(payload); + + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + }, + }); + } + function handleAttemptComplete( state: ServerState, ws: DevSocket, @@ -158,8 +178,7 @@ const createSocketAPI = ( const { ref, topic, payload } = evt; const { dataclip } = payload; - // TODO use proper logger - logger?.info('Completed attempted ', attemptId); + logger?.info('Completed attempt ', attemptId); logger?.debug(dataclip); state.pending[attemptId].status = 'complete'; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 79f4c30be..56f7968e7 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -1,24 +1,24 @@ import { EventEmitter } from 'node:events'; import Koa from 'koa'; -import URL from 'node:url'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; -import websockify from 'koa-websocket'; -import route from 'koa-route'; import createLogger, { createMockLogger, LogLevel, Logger, + JSONLog, } from '@openfn/logger'; -import createServer from './socket-server'; -import createAPI from './api'; import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; -import { Attempt } from '../../types'; export const API_PREFIX = '/api/1'; +export type AttemptState = { + status: 'queued' | 'started' | 'complete'; + logs: JSONLog[]; +}; + export type ServerState = { queue: AttemptId[]; @@ -33,7 +33,7 @@ export type ServerState = { // Tracking state of known attempts // TODO include the rtm id and token - pending: Record; + pending: Record; // Track all completed attempts here results: Record; diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index b2ab26d38..08b496d7f 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -6,12 +6,14 @@ import phx from 'phoenix-channels'; import { attempts, credentials, dataclips } from './data'; import { ATTEMPT_COMPLETE, + ATTEMPT_LOG, CLAIM, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, } from '../../src/events'; import type { Attempt } from '../../src/types'; +import { JSONLog } from '@openfn/logger'; const endpoint = 'ws://localhost:7777/api'; @@ -248,14 +250,37 @@ test.serial('complete an attempt through the attempt channel', async (t) => { .push(ATTEMPT_COMPLETE, { dataclip: { answer: 42 } }) .receive('ok', () => { const { pending, results } = server.getState(); - t.deepEqual(pending[a.id], { status: 'complete' }); + t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); t.deepEqual(results[a.id], { answer: 42 }); done(); }); }); }); -test.serial('unusubscribe after attempt complete', async (t) => { +test.serial('logs are saved and acknowledged', async (t) => { + return new Promise(async (done) => { + server.registerAttempt(attempt1); + server.startAttempt(attempt1.id); + + const log = { + attempt_id: attempt1.id, + level: 'info', + name: 'R/T', + message: ['Did the thing'], + time: new Date().getTime(), + } as JSONLog; + + const channel = await join(`attempt:${attempt1.id}`); + channel.push(ATTEMPT_LOG, log).receive('ok', () => { + const { pending } = server.getState(); + const [savedLog] = pending[attempt1.id].logs; + t.deepEqual(savedLog, log); + done(); + }); + }); +}); + +test.serial('unsubscribe after attempt complete', async (t) => { return new Promise(async (done) => { const a = attempt1; server.registerAttempt(a); From db9ecc2bf129774f828376bc78fa3a7ab670b801 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 11:41:31 +0100 Subject: [PATCH 082/232] rtm: update tests --- .../test/mock/socket-server.test.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index 4cd21a2ff..9b3ecd3b5 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -14,6 +14,7 @@ const wait = (duration = 10) => test.beforeEach(() => { messages = []; + // @ts-ignore I don't care about missing server options here server = createServer({ onMessage: (evt) => messages.push(evt) }); socket = new phx.Socket('ws://localhost:8080'); @@ -41,7 +42,7 @@ test.serial('send a message', async (t) => { return new Promise((resolve) => { const channel = socket.channel('x', {}); - server.listenToChannel('x', (event, payload) => { + server.listenToChannel('x', (_ws, { payload, event }) => { t.is(event, 'hello'); t.deepEqual(payload, { x: 1 }); @@ -116,19 +117,18 @@ test.serial('wait for message', async (t) => { t.truthy(result); }); -test.serial.only('onMessage', (t) => { +test.serial('onMessage', (t) => { return new Promise((done) => { const channel = socket.channel('x', {}); channel.join().receive('ok', async () => { - t.is(messages.length, 1) - t.is(messages[0].event, 'phx_join') + t.is(messages.length, 1); + t.is(messages[0].event, 'phx_join'); channel.push('hello', { x: 1 }); await server.waitForMessage('x', 'hello'); - t.is(messages.length, 2) - t.is(messages[1].event, 'hello') - done() - }) - }) - -}) \ No newline at end of file + t.is(messages.length, 2); + t.is(messages[1].event, 'hello'); + done(); + }); + }); +}); From abcb9f8b73355028a6bf87b29f45577a7b984605 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 11:59:19 +0100 Subject: [PATCH 083/232] rtm: restore waitForResult to lightning mock --- .../rtm-server/src/mock/lightning/api-dev.ts | 17 ++- .../src/mock/lightning/api-sockets.ts | 6 + .../src/mock/lightning/middleware.ts | 116 ------------------ .../rtm-server/src/mock/lightning/server.ts | 20 ++- packages/rtm-server/test/events.test.ts | 52 -------- .../rtm-server/test/mock/lightning.test.ts | 27 ++++ 6 files changed, 53 insertions(+), 185 deletions(-) delete mode 100644 packages/rtm-server/src/mock/lightning/middleware.ts delete mode 100644 packages/rtm-server/test/events.test.ts diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/rtm-server/src/mock/lightning/api-dev.ts index f87dbd32d..683cb9122 100644 --- a/packages/rtm-server/src/mock/lightning/api-dev.ts +++ b/packages/rtm-server/src/mock/lightning/api-dev.ts @@ -4,11 +4,12 @@ */ import Koa from 'koa'; import Router from '@koa/router'; -import { Logger } from '@openfn/logger'; +import { JSONLog, Logger } from '@openfn/logger'; import crypto from 'node:crypto'; import { Attempt } from '../../types'; import { ServerState } from './server'; +import { ATTEMPT_COMPLETE } from '../../events'; type LightningEvents = 'log' | 'attempt-complete'; @@ -44,6 +45,7 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { state.results[attempt.id] = {}; state.pending[attempt.id] = { status: 'queued', + logs: [], }; state.queue.push(attempt.id); }; @@ -55,13 +57,18 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { // Promise which returns when a workflow is complete app.waitForResult = (attemptId: string) => { return new Promise((resolve) => { - const handler = (evt: any) => { - if (evt.workflow_id === attemptId) { - state.events.removeListener('attempt-complete', handler); + const handler = (evt: { + attemptId: string; + dataclip: any; + logs: JSONLog[]; + }) => { + if (evt.attemptId === attemptId) { + state.events.removeListener(ATTEMPT_COMPLETE, handler); + resolve(evt); } }; - state.events.addListener('attempt-complete', handler); + state.events.addListener(ATTEMPT_COMPLETE, handler); }); }; diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index a65de70bf..1beba4443 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -184,6 +184,12 @@ const createSocketAPI = ( state.pending[attemptId].status = 'complete'; state.results[attemptId] = dataclip; + state.events.emit(ATTEMPT_COMPLETE, { + attemptId: attemptId, + dataclip, + logs: state.pending[attemptId].logs, + }); + ws.reply({ ref, topic, diff --git a/packages/rtm-server/src/mock/lightning/middleware.ts b/packages/rtm-server/src/mock/lightning/middleware.ts deleted file mode 100644 index dd555bc7c..000000000 --- a/packages/rtm-server/src/mock/lightning/middleware.ts +++ /dev/null @@ -1,116 +0,0 @@ -/// Thhis is basically deprecated -import Koa from 'koa'; -import type { ServerState } from './server'; -import { AttemptCompleteBody } from './api'; - -// basically an author handler -const shouldAcceptRequest = ( - state: ServerState, - jobId: string, - request: Koa.Request -) => { - const { results } = state; - if (request.body) { - const { rtm_id } = request.body; - return results[jobId] && results[jobId].rtmId === rtm_id; - } -}; - -export const unimplemented = (ctx: Koa.Context) => { - ctx.status = 501; -}; - -export const createClaim = (state: ServerState) => (ctx: Koa.Context) => { - const { queue } = state; - const { body } = ctx.request; - if (!body || !body.rtm_id) { - ctx.status = 400; - return; - } - const countRaw = ctx.request.query.count as unknown; - let count = 1; - if (countRaw) { - if (!isNaN(countRaw)) { - count = countRaw as number; - } else { - console.error('Failed to parse parameter countRaw'); - } - } - const payload = []; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - payload.push(queue.shift()); - count -= 1; - } - if (payload.length > 0) { - ctx.body = JSON.stringify(payload); - ctx.status = 200; - } else { - ctx.body = undefined; - ctx.status = 204; - } -}; - -export const createListNextJob = (state: ServerState) => (ctx: Koa.Context) => { - ctx.body = state.queue.map(({ id }) => id); - ctx.status = 200; -}; - -export const createGetCredential = - (state: ServerState) => (ctx: Koa.Context) => { - const { credentials } = state; - const cred = credentials[ctx.params.id]; - if (cred) { - ctx.body = JSON.stringify(cred); - ctx.status = 200; - } else { - ctx.status = 404; - } - }; - -export const createLog = (state: ServerState) => (ctx: Koa.Context) => { - const { events } = state; - if (shouldAcceptRequest(state, ctx.params.id, ctx.request)) { - const data = ctx.request.body; - const event = { - id: ctx.params.id, - logs: data.logs, - }; - - events.emit('log', event); - - ctx.status = 200; - } else { - ctx.status = 400; - } -}; - -export const createComplete = - (state: ServerState) => - ( - ctx: Koa.ParameterizedContext< - Koa.DefaultState, - Koa.DefaultContext, - AttemptCompleteBody - > - ) => { - const { results, events } = state; - const { state: resultState, rtm_id } = ctx.request.body; - - if (shouldAcceptRequest(state, ctx.params.id, ctx.request)) { - results[ctx.params.id].state = resultState; - - events.emit('attempt-complete', { - rtm_id, - workflow_id: ctx.params.id, - state: resultState, - }); - - ctx.status = 200; - } else { - // Unexpected result - ctx.status = 400; - } - }; diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/rtm-server/src/mock/lightning/server.ts index 56f7968e7..ad536f696 100644 --- a/packages/rtm-server/src/mock/lightning/server.ts +++ b/packages/rtm-server/src/mock/lightning/server.ts @@ -72,14 +72,6 @@ const createLightningServer = (options: LightningOptions = {}) => { const server = app.listen(port); logger.info('Listening on ', port); - // Setup the websocket API - const api = createWebSocketAPI( - state, - '/api', - server, - options.logger && logger - ); - // Only create a http logger if there's a top-level logger passed // This is a bit flaky really but whatever if (options.logger) { @@ -88,10 +80,14 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(klogger); } - // Mock API endpoints - // TODO should we keep the REST interface for local debug? - // Maybe for the read-only stuff (like get all attempts) - // app.use(createAPI(state)); + // Setup the websocket API + const api = createWebSocketAPI( + state, + '/api', + server, + options.logger && logger + ); + app.use(createDevAPI(app as any, state, logger, api)); app.destroy = () => { diff --git a/packages/rtm-server/test/events.test.ts b/packages/rtm-server/test/events.test.ts deleted file mode 100644 index 45ef2edc6..000000000 --- a/packages/rtm-server/test/events.test.ts +++ /dev/null @@ -1,52 +0,0 @@ -// TODO This can all be removed -/** - * Unit tests on events published by the rtm-server - * No lightning involved here - */ -import test from 'ava'; -import createRTMServer from '../src/server'; -import createMockRTM from '../src/mock/runtime-manager'; -import { waitForEvent } from './util'; - -const str = (obj: object) => JSON.stringify(obj); - -let server; - -test.before(() => { - const rtm = createMockRTM(); - server = createRTMServer(rtm, { port: 2626 }); -}); - -test.serial.skip( - 'trigger a workflow-start event when execution starts', - async (t) => { - server.execute({ - id: 'a', - triggers: [{ id: 't', next: { b: true } }], - jobs: [{ id: 'j' }], - }); - - const evt = await waitForEvent(server, 'workflow-start'); - t.truthy(evt); - t.is(evt.id, 'a'); - - // TODO what goes in this event? - // Test more carefully - } -); - -test.serial.skip( - 'trigger a workflow-complete event when execution completes', - async (t) => { - server.execute({ - id: 'a', - triggers: [{ id: 't', next: { b: true } }], - jobs: [{ id: 'j', body: str({ answer: 42 }) }], - }); - - const evt = await waitForEvent(server, 'workflow-complete'); - t.truthy(evt); - t.is(evt.id, 'a'); - t.deepEqual(evt.state, { answer: 42 }); - } -); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 08b496d7f..98e9afa35 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -323,3 +323,30 @@ test.serial('get dataclip through the attempt channel', async (t) => { }); }); }); + +// TODO test that all events are proxied out to server.on + +test.serial( + 'waitForResult should return logs and dataclip when an attempt is completed', + async (t) => { + return new Promise(async (done) => { + server.startAttempt(attempt1.id); + server.addDataclip('d', dataclips['d']); + const result = { answer: 42 }; + + server + .waitForResult(attempt1.id) + .then(({ attemptId, dataclip, logs }) => { + t.is(attemptId, attempt1.id); + t.deepEqual(result, dataclip); + t.deepEqual(logs, []); + done(); + }); + + const channel = await join(`attempt:${attempt1.id}`); + channel.push(ATTEMPT_COMPLETE, { dataclip: result }); + }); + } +); + +// test.serial('getLogs should return logs', async (t) => {}); From 247fbf5dee998a657a383d87173ca7d9e7f5b1b2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 13:00:32 +0100 Subject: [PATCH 084/232] rtm: tidy up the server a bit --- packages/rtm-server/src/api/execute.ts | 65 ++++++++++-------- .../rtm-server/src/mock/runtime-manager.ts | 7 +- packages/rtm-server/test/api/execute.test.ts | 67 +++++++++++++++++-- 3 files changed, 98 insertions(+), 41 deletions(-) diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index d0ca77c97..d535d96d5 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -14,6 +14,7 @@ import convertAttempt from '../util/convert-attempt'; import { ATTEMPT_COMPLETE, ATTEMPT_LOG, + ATTEMPT_START, GET_ATTEMPT, RUN_COMPLETE, RUN_START, @@ -25,6 +26,10 @@ export type AttemptState = { activeRun?: string; activeJob?: string; plan: ExecutionPlan; + // final state/dataclip + result?: any; + + // TODO status? }; type Channel = typeof phx.Channel; @@ -76,14 +81,20 @@ export function onJobComplete( delete state.activeJob; } -export function onWorkflowComplete(channel: Channel, state, evt) { +export function onWorkflowStart(channel: Channel) { + channel.push(ATTEMPT_START); +} + +export function onWorkflowComplete( + channel: Channel, + state: AttemptState, + evt: any +) { + state.result = evt.state; + channel.push(ATTEMPT_COMPLETE, { - // no point in publishing the workflow id - // TODO include final data dataclip: evt.state, }); - - // TODO should I mark the state? Is there any point? } export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { @@ -102,7 +113,6 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { export async function prepareAttempt(channel: Channel) { // first we get the attempt body through the socket const attemptBody = (await getWithReply(channel, GET_ATTEMPT)) as Attempt; - console.log(attemptBody); // then we generate the execution plan const plan = convertAttempt(attemptBody); @@ -125,29 +135,26 @@ async function loadCredential(ws, attemptId, stateId) {} // TODO actually this now is a Workflow or Execution plan // It's not an attempt anymore export function execute(channel: Channel, rtm, plan: ExecutionPlan) { - // TODO add proper logger (maybe channel, rtm and logger comprise a context object) - console.log('execute', plan); - // tracking state for this attempt - const state: AttemptState = { - //attempt, // keep this on the state so that anyone can access it - plan, - }; - - // listen to rtm events - // what if I can do this - // this is super declarative - // TODO is there any danger of events coming through out of order? - // what if onJoblog takes 1 second to finish and before the runId is set, onJobLog comes through? - - // TODO - // const context = { channel, state } + return new Promise((resolve) => { + // TODO add proper logger (maybe channel, rtm and logger comprise a context object) + // tracking state for this attempt + const state: AttemptState = { + plan, + }; + + // TODO + // const context = { channel, state, logger } + + rtm.listen(plan.id, { + 'job-start': (evt) => onJobStart(channel, state, evt), + 'job-complete': (evt) => onJobComplete(channel, state, evt), + 'job-log': (evt) => onJobLog(channel, state, evt), + 'workflow-complete': (evt) => { + onWorkflowComplete(channel, state, evt); + resolve(evt.state); + }, + }); - rtm.listen(plan.id, { - 'job-start': (evt) => onJobStart(channel, state, evt), - 'job-complete': (evt) => onJobComplete(channel, state, evt), - 'job-log': (evt) => onJobLog(channel, state, evt), - 'workflow-complete': (evt) => onWorkflowComplete(channel, state, evt), + rtm.execute(plan); }); - - rtm.execute(plan); } diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index ffc5cb617..f3eba0105 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -71,7 +71,7 @@ function createMock( const listeners: Record = {}; const dispatch = (type: RTMEvent, args?: any) => { - console.log(' > ', type, args); + // console.log(' > ', type, args); if (args.workflowId) { listeners[args.workflowId]?.[type]?.(args); } @@ -137,13 +137,10 @@ function createMock( // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = (xplan: ExecutionPlan) => { - console.log('[mockrtm] execute'); - console.log(xplan); const { id, jobs } = xplan; const workflowId = id; activeWorkflows[id!] = true; setTimeout(() => { - console.log('[mockrtm] start'); dispatch('workflow-start', { workflowId }); setTimeout(async () => { let state = {}; @@ -153,8 +150,6 @@ function createMock( } setTimeout(() => { delete activeWorkflows[id!]; - console.log('[mockrtm] complete'); - console.log(state); dispatch('workflow-complete', { workflowId, state }); // TODO on workflow complete we should maybe tidy the listeners? // Doesn't really matter in the mock though diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index d273ada99..d8c63eeff 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -6,6 +6,8 @@ import { RUN_START, RUN_COMPLETE, ATTEMPT_LOG, + ATTEMPT_START, + ATTEMPT_COMPLETE, } from '../../src/events'; import { prepareAttempt, @@ -13,6 +15,9 @@ import { onJobComplete, onJobLog, execute, + onWorkflowStart, + onWorkflowComplete, + AttemptState, } from '../../src/api/execute'; import createMockRTM from '../../src/mock/runtime-manager'; import { attempts } from '../mock/data'; @@ -64,7 +69,7 @@ test('jobStart should set a run id and active job on state', async (t) => { const state = { plan, - }; + } as AttemptState; const channel = mockChannel({}); @@ -81,7 +86,7 @@ test('jobStart should send a run:start event', async (t) => { const state = { plan, - }; + } as AttemptState; const channel = mockChannel({ [RUN_START]: (evt) => { @@ -104,7 +109,7 @@ test('jobEnd should clear the run id and active job on state', async (t) => { plan, activeJob: jobId, activeRun: 'b', - }; + } as AttemptState; const channel = mockChannel({}); @@ -123,7 +128,7 @@ test('jobComplete should send a run:complete event', async (t) => { plan, activeJob: jobId, activeRun: 'b', - }; + } as AttemptState; const channel = mockChannel({ [RUN_COMPLETE]: (evt) => { @@ -157,7 +162,7 @@ test('jobLog should should send a log event outside a run', async (t) => { const state = { plan, // No active run - }; + } as AttemptState; const channel = mockChannel({ [ATTEMPT_LOG]: (evt) => { @@ -186,7 +191,7 @@ test('jobLog should should send a log event inside a run', async (t) => { plan, activeJob: jobId, activeRun: 'b', - }; + } as AttemptState; const channel = mockChannel({ [ATTEMPT_LOG]: (evt) => { @@ -203,6 +208,56 @@ test('jobLog should should send a log event inside a run', async (t) => { }); }); +test('workflowStart should send an empty attempt:start event', async (t) => { + return new Promise((done) => { + const channel = mockChannel({ + [ATTEMPT_START]: () => { + t.pass(); + + done(); + }, + }); + + onWorkflowStart(channel); + }); +}); + +test('workflowComplete should send an attempt:complete event', async (t) => { + return new Promise((done) => { + const state = {} as AttemptState; + + const result = { answer: 42 }; + + const channel = mockChannel({ + [ATTEMPT_COMPLETE]: (evt) => { + t.deepEqual(evt.dataclip, result); + t.deepEqual(state.result, result); + + done(); + }, + }); + + onWorkflowComplete(channel, state, { state: result }); + }); +}); + +test('execute should return the final result', async (t) => { + const channel = mockChannel(); + const rtm = createMockRTM(); + + const plan = { + id: 'a', + jobs: [ + { + expression: JSON.stringify({ done: true }), + }, + ], + }; + + const result = await execute(channel, rtm, plan); + + t.deepEqual(result, { done: true }); +}); // TODO test the whole execute workflow // run this against the mock - this just ensures that execute From c814323be9d5dcb37cbc0156a25eec855e08931e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 16:00:42 +0100 Subject: [PATCH 085/232] rtm: testing --- packages/rtm-server/src/api/execute.ts | 3 +- .../rtm-server/src/mock/runtime-manager.ts | 29 +++--- packages/rtm-server/test/api/execute.test.ts | 92 ++++++++++++++++--- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index d535d96d5..7695efa6e 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -146,9 +146,10 @@ export function execute(channel: Channel, rtm, plan: ExecutionPlan) { // const context = { channel, state, logger } rtm.listen(plan.id, { + 'workflow-start': (evt) => onWorkflowStart(channel), 'job-start': (evt) => onJobStart(channel, state, evt), 'job-complete': (evt) => onJobComplete(channel, state, evt), - 'job-log': (evt) => onJobLog(channel, state, evt), + log: (evt) => onJobLog(channel, state, evt), 'workflow-complete': (evt) => { onWorkflowComplete(channel, state, evt); resolve(evt.state); diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index f3eba0105..c0457dbee 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -77,16 +77,11 @@ function createMock( } // TODO add performance metrics to every event? bus.emit(type, args); - - // TOOD return an unsubscribe API? }; - const on = (event: RTMEvent, fn: (evt: any) => void) => { - bus.on(event, fn); - }; - const once = (event: RTMEvent, fn: (evt: any) => void) => { - bus.once(event, fn); - }; + const on = (event: RTMEvent, fn: (evt: any) => void) => bus.on(event, fn); + + const once = (event: RTMEvent, fn: (evt: any) => void) => bus.once(event, fn); // Listens to events for a particular workflow/execution plan // TODO: Listeners will be removed when the plan is complete (?) @@ -97,23 +92,33 @@ function createMock( listeners[planId] = events; }; - const executeJob = async (workflowId, job: JobPlan, initialState = {}) => { + const executeJob = async ( + workflowId: string, + job: JobPlan, + initialState = {} + ) => { // TODO maybe lazy load the job from an id - const { id, expression, configuration } = job; + const { id, expression, configuration, state } = job; const jobId = id; if (typeof configuration === 'string') { // Fetch the credential but do nothing with it // Maybe later we use it to assemble state await resolvers.credentials(configuration); } + if (typeof state === 'string') { + // TODO right now we lazy load any state object + // but maybe we need to just do initial state? + await resolvers.state(state); + } // Does a job reallly need its own id...? Maybe. const runId = getNewJobId(); // Get the job details from lightning // start instantly and emit as it goes - dispatch('job-start', { workflowId, jobId }); + dispatch('job-start', { workflowId, jobId, runId }); + dispatch('log', { workflowId, jobId, message: ['Running job ' + jobId] }); let state = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { @@ -129,7 +134,7 @@ function createMock( // Do nothing, it's fine } - dispatch('job-complete', { workflowId, jobId, state }); + dispatch('job-complete', { workflowId, jobId, state, runId }); return state; }; diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index d8c63eeff..f5e11e46b 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -8,6 +8,8 @@ import { ATTEMPT_LOG, ATTEMPT_START, ATTEMPT_COMPLETE, + GET_CREDENTIAL, + GET_DATACLIP, } from '../../src/events'; import { prepareAttempt, @@ -258,21 +260,88 @@ test('execute should return the final result', async (t) => { t.deepEqual(result, { done: true }); }); -// TODO test the whole execute workflow -// run this against the mock - this just ensures that execute -// binds all the events -test.skip('execute should call all events', async (t) => { - const events = {}; +// TODO this is more of an RTM test really, but worth having I suppose +test('execute should lazy-load a credential', async (t) => { + let didCallCredentials = false; - const rtm = createMockRTM(); + const channel = mockChannel(); + const rtm = createMockRTM('rtm', { + credentials: (id) => { + t.truthy(id); + didCallCredentials = true; + return {}; + }, + }); - const channel = mockChannel({ - [ATTEMPT_LOG]: (evt) => { - events[ATTEMPT_LOG] = evt; + const plan = { + id: 'a', + jobs: [ + { + configuration: 'abc', + expression: JSON.stringify({ done: true }), + }, + ], + }; + + await execute(channel, rtm, plan); + + t.true(didCallCredentials); +}); + +// TODO this is more of an RTM test really, but worth having I suppose +test('execute should lazy-load initial state', async (t) => { + let didCallState = false; + + const channel = mockChannel(); + const rtm = createMockRTM('rtm', { + state: (id) => { + t.truthy(id); + didCallState = true; + return {}; }, }); + const plan = { + id: 'a', + jobs: [ + { + state: 'abc', + expression: JSON.stringify({ done: true }), + }, + ], + }; + + await execute(channel, rtm, plan); + + t.true(didCallState); +}); + +test('execute should call all events on the socket', async (t) => { + const events = {}; + + const rtm = createMockRTM(); + + const toEventMap = (obj, evt: string) => { + obj[evt] = (e) => { + events[evt] = e || true; + }; + return obj; + }; + + const allEvents = [ + // Note that these are listed in order but order isn not tested + // GET_CREDENTIAL, // TODO not implementated yet + // GET_DATACLIP, // TODO not implementated yet + ATTEMPT_START, + RUN_START, + ATTEMPT_LOG, + RUN_COMPLETE, + ATTEMPT_COMPLETE, + ]; + + const channel = mockChannel(allEvents.reduce(toEventMap, {})); + const plan = { id: 'attempt-1', jobs: [ @@ -285,11 +354,12 @@ test.skip('execute should call all events', async (t) => { ], }; - const result = await execute(channel, rtm, plan); + await execute(channel, rtm, plan); // check result is what we expect // Check that events were passed to the socket // This is deliberately crude - t.truthy(events[ATTEMPT_LOG]); + console.log(events); + t.assert(allEvents.every((e) => events[e])); }); From 4d712c7e5c090c264ec30315414b165947f93f39 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 16:18:43 +0100 Subject: [PATCH 086/232] rtm: integrate lazy resolvers to use the channel --- packages/rtm-server/src/api/execute.ts | 80 +++++++++---------- .../rtm-server/src/mock/runtime-manager.ts | 17 ++-- packages/rtm-server/src/types.d.ts | 4 + .../rtm-server/src/util/get-with-reply.ts | 9 +++ packages/rtm-server/src/util/index.ts | 3 +- packages/rtm-server/test/api/execute.test.ts | 45 ++++++++--- 6 files changed, 96 insertions(+), 62 deletions(-) create mode 100644 packages/rtm-server/src/util/get-with-reply.ts diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 7695efa6e..5fa9627f9 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -5,7 +5,6 @@ // You can almost read this as a binding function and a bunch of handlers // it isn't an actual worker, but a BRIDGE between a worker and lightning import crypto from 'node:crypto'; -import phx from 'phoenix-channels'; import { JSONLog } from '@openfn/logger'; import convertAttempt from '../util/convert-attempt'; // this managers the worker @@ -16,11 +15,14 @@ import { ATTEMPT_LOG, ATTEMPT_START, GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, RUN_COMPLETE, RUN_START, } from '../events'; -import { Attempt } from '../types'; +import { Attempt, Channel } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; +import { getWithReply } from '../util'; export type AttemptState = { activeRun?: string; @@ -32,19 +34,41 @@ export type AttemptState = { // TODO status? }; -type Channel = typeof phx.Channel; +// pass a web socket connected to the attempt channel +// this thing will do all the work +export function execute(channel: Channel, rtm, plan: ExecutionPlan) { + return new Promise((resolve) => { + // TODO add proper logger (maybe channel, rtm and logger comprise a context object) + // tracking state for this attempt + const state: AttemptState = { + plan, + }; -// TODO move to util + // TODO + // const context = { channel, state, logger } -const getWithReply = (channel: Channel, event: string, payload?: any) => - new Promise((resolve) => { - channel.push(event, payload).receive('ok', (evt: any) => { - resolve(evt); + rtm.listen(plan.id, { + 'workflow-start': (evt) => onWorkflowStart(channel), + 'job-start': (evt) => onJobStart(channel, state, evt), + 'job-complete': (evt) => onJobComplete(channel, state, evt), + log: (evt) => onJobLog(channel, state, evt), + 'workflow-complete': (evt) => { + onWorkflowComplete(channel, state, evt); + resolve(evt.state); + }, }); - // TODO handle errors amd timeouts too + + const resolvers = { + state: (id: string) => loadState(channel, id), + credential: (id: string) => loadCredential(channel, id), + }; + + rtm.execute(plan, resolvers); }); +} // TODO maybe move all event handlers into api/events/* + export function onJobStart( channel: Channel, state: AttemptState, @@ -124,38 +148,10 @@ export async function prepareAttempt(channel: Channel) { // then we call the excute function. Or return the promise and let someone else do that } -// These are the functions that lazy load data from lightning -// Is it appropriate first join the channel? Should there be some pooling? -async function loadState(ws, attemptId, stateId) {} - -async function loadCredential(ws, attemptId, stateId) {} - -// pass a web socket connected to the attempt channel -// this thing will do all the work -// TODO actually this now is a Workflow or Execution plan -// It's not an attempt anymore -export function execute(channel: Channel, rtm, plan: ExecutionPlan) { - return new Promise((resolve) => { - // TODO add proper logger (maybe channel, rtm and logger comprise a context object) - // tracking state for this attempt - const state: AttemptState = { - plan, - }; - - // TODO - // const context = { channel, state, logger } - - rtm.listen(plan.id, { - 'workflow-start': (evt) => onWorkflowStart(channel), - 'job-start': (evt) => onJobStart(channel, state, evt), - 'job-complete': (evt) => onJobComplete(channel, state, evt), - log: (evt) => onJobLog(channel, state, evt), - 'workflow-complete': (evt) => { - onWorkflowComplete(channel, state, evt); - resolve(evt.state); - }, - }); +export async function loadState(channel: Channel, stateId: string) { + return getWithReply(channel, GET_DATACLIP, { dataclip_id: stateId }); +} - rtm.execute(plan); - }); +export async function loadCredential(channel: Channel, credentialId: string) { + return getWithReply(channel, GET_CREDENTIAL, { credential_id: credentialId }); } diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/rtm-server/src/mock/runtime-manager.ts index c0457dbee..841630271 100644 --- a/packages/rtm-server/src/mock/runtime-manager.ts +++ b/packages/rtm-server/src/mock/runtime-manager.ts @@ -62,10 +62,7 @@ let autoServerId = 0; // The credential of course is the hard bit const assembleState = () => {}; -function createMock( - serverId?: string, - resolvers: LazyResolvers = mockResolvers -) { +function createMock(serverId?: string) { const activeWorkflows = {} as Record; const bus = new EventEmitter(); const listeners: Record = {}; @@ -95,7 +92,8 @@ function createMock( const executeJob = async ( workflowId: string, job: JobPlan, - initialState = {} + initialState = {}, + resolvers: LazyResolvers = mockResolvers ) => { // TODO maybe lazy load the job from an id const { id, expression, configuration, state } = job; @@ -103,7 +101,7 @@ function createMock( if (typeof configuration === 'string') { // Fetch the credential but do nothing with it // Maybe later we use it to assemble state - await resolvers.credentials(configuration); + await resolvers.credential(configuration); } if (typeof state === 'string') { // TODO right now we lazy load any state object @@ -141,7 +139,10 @@ function createMock( // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity - const execute = (xplan: ExecutionPlan) => { + const execute = ( + xplan: ExecutionPlan, + resolvers: LazyResolvers = mockResolvers + ) => { const { id, jobs } = xplan; const workflowId = id; activeWorkflows[id!] = true; @@ -151,7 +152,7 @@ function createMock( let state = {}; // Trivial job reducer in our mock for (const job of jobs) { - state = await executeJob(id, job, state); + state = await executeJob(id, job, state, resolvers); } setTimeout(() => { delete activeWorkflows[id!]; diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index c36cfdec3..2b421504c 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -1,3 +1,5 @@ +import phx from 'phoenix-channels'; + export type Credential = Record; export type State = { @@ -51,6 +53,8 @@ export type CancelablePromise = Promise & { cancel: () => void; }; +export type Channel = typeof phx.Channel; + // type RuntimeExecutionPlanID = string; // type JobEdge = { diff --git a/packages/rtm-server/src/util/get-with-reply.ts b/packages/rtm-server/src/util/get-with-reply.ts new file mode 100644 index 000000000..cc1dd15f9 --- /dev/null +++ b/packages/rtm-server/src/util/get-with-reply.ts @@ -0,0 +1,9 @@ +import { Channel } from '../types'; + +export default (channel: Channel, event: string, payload?: any) => + new Promise((resolve) => { + channel.push(event, payload).receive('ok', (evt: any) => { + resolve(evt); + }); + // TODO handle errors and timeouts too + }); diff --git a/packages/rtm-server/src/util/index.ts b/packages/rtm-server/src/util/index.ts index 1a186e17a..a9e402621 100644 --- a/packages/rtm-server/src/util/index.ts +++ b/packages/rtm-server/src/util/index.ts @@ -1,4 +1,5 @@ import convertAttempt from './convert-attempt'; import tryWithBackoff from './try-with-backoff'; +import getWithReply from './get-with-reply'; -export { convertAttempt, tryWithBackoff }; +export { convertAttempt, tryWithBackoff, getWithReply }; diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index f5e11e46b..40b14d173 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -20,15 +20,13 @@ import { onWorkflowStart, onWorkflowComplete, AttemptState, + loadState, + loadCredential, } from '../../src/api/execute'; import createMockRTM from '../../src/mock/runtime-manager'; import { attempts } from '../mock/data'; import { mockChannel } from '../util'; -// This is a fake/mock websocket used by mocks - -// TODO throw in the handler to get an error? - test('prepareAttempt should get the attempt body', async (t) => { const attempt = attempts['attempt-1']; let didCallGetAttempt = false; @@ -243,6 +241,32 @@ test('workflowComplete should send an attempt:complete event', async (t) => { }); }); +// TODO what if an error? +test('loadState should fetch a dataclip', async (t) => { + const channel = mockChannel({ + [GET_DATACLIP]: ({ dataclip_id }) => { + t.is(dataclip_id, 'xyz'); + return { data: {} }; + }, + }); + + const state = await loadState(channel, 'xyz'); + t.deepEqual(state, { data: {} }); +}); + +// TODO what if an error? +test('loadCredential should fetch a credential', async (t) => { + const channel = mockChannel({ + [GET_CREDENTIAL]: ({ credential_id }) => { + t.is(credential_id, 'jfk'); + return { apiKey: 'abc' }; + }, + }); + + const state = await loadCredential(channel, 'jfk'); + t.deepEqual(state, { apiKey: 'abc' }); +}); + test('execute should return the final result', async (t) => { const channel = mockChannel(); const rtm = createMockRTM(); @@ -265,14 +289,14 @@ test('execute should return the final result', async (t) => { test('execute should lazy-load a credential', async (t) => { let didCallCredentials = false; - const channel = mockChannel(); - const rtm = createMockRTM('rtm', { - credentials: (id) => { + const channel = mockChannel({ + [GET_CREDENTIAL]: (id) => { t.truthy(id); didCallCredentials = true; return {}; }, }); + const rtm = createMockRTM('rtm'); const plan = { id: 'a', @@ -293,14 +317,14 @@ test('execute should lazy-load a credential', async (t) => { test('execute should lazy-load initial state', async (t) => { let didCallState = false; - const channel = mockChannel(); - const rtm = createMockRTM('rtm', { - state: (id) => { + const channel = mockChannel({ + [GET_DATACLIP]: (id) => { t.truthy(id); didCallState = true; return {}; }, }); + const rtm = createMockRTM('rtm'); const plan = { id: 'a', @@ -360,6 +384,5 @@ test('execute should call all events on the socket', async (t) => { // Check that events were passed to the socket // This is deliberately crude - console.log(events); t.assert(allEvents.every((e) => events[e])); }); From e0c8966726732a9a7015576999459983414f1d36 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 18:04:45 +0100 Subject: [PATCH 087/232] rtm: strognger typings around lightning events (plus some refactoring) --- packages/rtm-server/src/events.ts | 17 ++++++++--------- .../src/mock/lightning/api-sockets.ts | 18 +++++++++++++----- .../src/mock/lightning/socket-server.ts | 19 ++++++++++++------- packages/rtm-server/src/server.ts | 5 ++++- .../rtm-server/test/mock/lightning.test.ts | 2 +- .../test/mock/runtime-manager.test.ts | 18 +++++++++++++----- packages/rtm-server/test/server.test.ts | 6 +++--- packages/rtm-server/test/util.ts | 3 +++ 8 files changed, 57 insertions(+), 31 deletions(-) diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index d2dff5a8c..bc0ecfa44 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -1,19 +1,18 @@ // track socket event names as constants to keep refactoring easier export const CLAIM = 'attempt:claim'; +// this is the lightning payload +export type CLAIM_PAYLOAD = { demand?: number }; +export type CLAIM_REPLY_PAYLOAD = Array<{ id: string; token?: string }>; -// TODO does each worker connect to its own channel, ensuring a private claim steeam? -// or is there a shared Workers channel - -// claim reply needs to include the id of the server and the attempt -export const CLAIM_REPLY = 'attempt:claim_reply'; // { server_id: 1, attempt_id: 'a1' } - -// All attempt events are in a dedicated channel for that event - -// or attempt_get ? I think there are several getters so maybe this makes sense export const GET_ATTEMPT = 'fetch:attempt'; +export type GET_ATTEMPT_PAYLOAD = undefined; // no payload + export const GET_CREDENTIAL = 'fetch:credential'; +// export type GET_CREDENTIAL_PAYLOAD = + export const GET_DATACLIP = 'fetch:dataclip'; +// export type GET_DATACLIP_PAYLOAD = export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index 1beba4443..f0c2848e4 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -8,11 +8,14 @@ import { extractAttemptId } from './util'; import createPheonixMockSocketServer, { DevSocket, PhoenixEvent, + PhoenixReply, } from './socket-server'; import { ATTEMPT_COMPLETE, ATTEMPT_LOG, CLAIM, + CLAIM_PAYLOAD, + CLAIM_REPLY_PAYLOAD, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, @@ -47,7 +50,8 @@ const createSocketAPI = ( }); wss.registerEvents('attempts:queue', { - [CLAIM]: (ws, event) => pullClaim(state, ws, event), + [CLAIM]: (ws, event: PhoenixEvent) => + pullClaim(state, ws, event), }); const startAttempt = (attemptId: string) => { @@ -83,7 +87,11 @@ const createSocketAPI = ( // pull claim will try and pull a claim off the queue, // and reply with the response // the reply ensures that only the calling worker will get the attempt - function pullClaim(state: ServerState, ws: DevSocket, evt) { + function pullClaim( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { const { ref, topic } = evt; const { queue } = state; let count = 1; @@ -91,16 +99,16 @@ const createSocketAPI = ( const payload = { status: 'ok', response: [], - }; + } as PhoenixReply['payload']; while (count > 0 && queue.length) { // TODO assign the worker id to the attempt // Not needed by the mocks at the moment const next = queue.shift(); - payload.response.push(next); + payload.response.push({ id: next! }); count -= 1; - startAttempt(next); + startAttempt(next!); } if (payload.response.length) { logger?.info(`Claiming ${payload.response.length} attempts`); diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 499f4baf3..a10478ba3 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -15,7 +15,7 @@ type Topic = string; // websocket with a couple of dev-friendly APIs export type DevSocket = WebSocket & { - reply: (evt: Pick) => void; + reply: (evt: PhoenixReply) => void; sendJSON: ({ event, topic, ref }: PhoenixEvent) => void; }; @@ -23,7 +23,16 @@ export type PhoenixEvent

= { topic: Topic; event: string; payload?: P; - ref?: string; + ref: string; +}; + +export type PhoenixReply = { + topic: Topic; + payload: { + status: 'ok' | 'error' | 'timeout'; + response: R; + }; + ref: string; }; type EventHandler = (ws: DevSocket, event: PhoenixEvent) => void; @@ -99,11 +108,7 @@ function createServer({ wsServer.on('connection', function (ws: DevSocket, _req: any) { logger?.info('new client connected'); - ws.reply = ({ - ref, - topic, - payload, - }: Pick) => { + ws.reply = ({ ref, topic, payload }: PhoenixReply) => { logger?.debug( `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) ); diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 99a95ad8a..4e1e21485 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -5,7 +5,6 @@ import phx from 'phoenix-channels'; import { createMockLogger, Logger } from '@openfn/logger'; import createRestAPI from './api-rest'; -import convertAttempt from './util/convert-attempt'; import startWorkloop from './api/workloop'; import { execute, prepareAttempt } from './api/execute'; @@ -39,10 +38,14 @@ export const connectToLightning = ( channel .join() .receive('ok', () => { + console.log('ok!'); done([socket, channel]); }) .receive('error', (e) => { console.log('ERROR', err); + }) + .receive('timeout', (e) => { + console.log('TIMEOUT', err); }); }); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 98e9afa35..d8d850f97 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -175,7 +175,7 @@ test.serial( channel.push(CLAIM).receive('ok', (response) => { t.truthy(response); t.is(response.length, 1); - t.is(response[0], 'attempt-1'); + t.deepEqual(response[0], { id: 'attempt-1' }); // ensure the server state has changed t.is(server.getQueueLength(), 0); diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/rtm-server/test/mock/runtime-manager.test.ts index 7dcc0da53..4fc0052df 100644 --- a/packages/rtm-server/test/mock/runtime-manager.test.ts +++ b/packages/rtm-server/test/mock/runtime-manager.test.ts @@ -103,9 +103,16 @@ test('mock should evaluate expressions as JSON', async (t) => { test('mock should dispatch log events when evaluating JSON', async (t) => { const rtm = create(); + const logs = []; + rtm.on('log', (l) => { + logs.push(l); + }); + rtm.execute(sampleWorkflow); - const evt = await waitForEvent(rtm, 'log'); - t.deepEqual(evt.message, ['Parsing expression as JSON state']); + await waitForEvent(rtm, 'workflow-complete'); + + t.deepEqual(logs[0].message, ['Running job j1']); + t.deepEqual(logs[1].message, ['Parsing expression as JSON state']); }); test('resolve credential before job-start if credential is a string', async (t) => { @@ -113,13 +120,14 @@ test('resolve credential before job-start if credential is a string', async (t) wf.jobs[0].configuration = 'x'; let didCallCredentials; - const credentials = async (_id) => { + const credential = async (_id) => { didCallCredentials = true; return {}; }; - const rtm = create('1', { credentials }); - rtm.execute(wf); + const rtm = create('1'); + // @ts-ignore + rtm.execute(wf, { credential }); await waitForEvent(rtm, 'job-start'); t.true(didCallCredentials); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index a5074c03c..70a908ccd 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; import WebSocket, { WebSocketServer } from 'ws'; -import createServer, { connectToLightning, startWorkloop } from '../src/server'; +import createServer, { connectToLightning } from '../src/server'; import createMockRTM from '../src/mock/runtime-manager'; import { mockChannel, mockSocket, sleep } from './util'; import { CLAIM } from '../src/events'; @@ -33,11 +33,11 @@ test.skip('healthcheck', async (t) => { }); // Not a very thorough test -test('connects to lightning', async (t) => { +test.only('connects to lightning', async (t) => { await connectToLightning('www', 'rtm', mockSocket); t.pass(); - // TODO connections to hte same socket.channel should share listners, so I think I can test the channel + // TODO connections to the same socket.channel should share listners, so I think I can test the channel }); // test('connects to websocket', (t) => { diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index 51fc8dddd..aff57b51e 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -64,6 +64,9 @@ export const mockChannel = (callbacks = {}) => { export const mockSocket = () => { const channels = {}; return { + onOpen: (callback) => { + setTimeout(callback, 1); + }, connect: () => { // noop // TODO maybe it'd be helpful to throw if the channel isn't connected? From 942d15daf8b940ed287150a25802fdd649054502 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 18:25:30 +0100 Subject: [PATCH 088/232] rtm: fix issue in mock socket --- packages/rtm-server/src/server.ts | 9 ++++----- packages/rtm-server/test/util.ts | 11 ++++++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 4e1e21485..d75c22ad6 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -38,14 +38,13 @@ export const connectToLightning = ( channel .join() .receive('ok', () => { - console.log('ok!'); done([socket, channel]); }) - .receive('error', (e) => { - console.log('ERROR', err); + .receive('error', (e: any) => { + console.log('ERROR', e); }) - .receive('timeout', (e) => { - console.log('TIMEOUT', err); + .receive('timeout', (e: any) => { + console.log('TIMEOUT', e); }); }); diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index aff57b51e..df8c52907 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -51,11 +51,16 @@ export const mockChannel = (callbacks = {}) => { }; }, join: () => { - return { + const receive = { receive: (status, callback) => { - callback(); + if (status === 'ok') { + setTimeout(() => callback(), 1); + } + // TODO error and timeout? + return receive; }, }; + return receive; }, }; return c; @@ -65,7 +70,7 @@ export const mockSocket = () => { const channels = {}; return { onOpen: (callback) => { - setTimeout(callback, 1); + setTimeout(() => callback(), 1); }, connect: () => { // noop From 3410941dca72ed628c00ebe7e73b6a43117848d4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 26 Sep 2023 18:29:44 +0100 Subject: [PATCH 089/232] rtm: skip test --- packages/rtm-server/test/integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rtm-server/test/integration.test.ts b/packages/rtm-server/test/integration.test.ts index 0962c03b6..267863eea 100644 --- a/packages/rtm-server/test/integration.test.ts +++ b/packages/rtm-server/test/integration.test.ts @@ -26,7 +26,7 @@ test.before(() => { }); // Really high level test -test.serial('process an attempt', async (t) => { +test.serial.skip('process an attempt', async (t) => { lng.enqueueAttempt({ id: 'a1', jobs: [ From 439c18ddf7c3be105fa155e88d6399e73c6c7b24 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 27 Sep 2023 12:47:16 +0100 Subject: [PATCH 090/232] rtm: add a whole bunch of typings --- packages/rtm-server/src/api/execute.ts | 34 ++++---- packages/rtm-server/src/events.ts | 50 ++++++++++-- .../src/mock/lightning/api-sockets.ts | 77 +++++++++++++------ .../src/mock/lightning/socket-server.ts | 4 +- packages/rtm-server/src/mock/sockets.ts | 55 +++++++++++++ packages/rtm-server/src/types.d.ts | 14 +++- packages/rtm-server/test/api/execute.test.ts | 2 +- packages/rtm-server/test/api/workloop.test.ts | 6 +- .../{util.test.ts => mock/sockets.test.ts} | 2 +- packages/rtm-server/test/util.ts | 57 +------------- 10 files changed, 194 insertions(+), 107 deletions(-) create mode 100644 packages/rtm-server/src/mock/sockets.ts rename packages/rtm-server/test/{util.test.ts => mock/sockets.test.ts} (96%) diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 5fa9627f9..4960656a8 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -12,13 +12,18 @@ import convertAttempt from '../util/convert-attempt'; import { ATTEMPT_COMPLETE, + ATTEMPT_COMPLETE_PAYLOAD, ATTEMPT_LOG, + ATTEMPT_LOG_PAYLOAD, ATTEMPT_START, + ATTEMPT_START_PAYLOAD, GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, RUN_COMPLETE, + RUN_COMPLETE_PAYLOAD, RUN_START, + RUN_START_PAYLOAD, } from '../events'; import { Attempt, Channel } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; @@ -74,18 +79,13 @@ export function onJobStart( state: AttemptState, jobId: string ) { - // generate a run id - // write it to state + // generate a run id and write it to state state.activeRun = crypto.randomUUID(); state.activeJob = jobId; - // post the correct event to the lightning via websocket - // do we need to wait for a response? Well, not yet. - - channel.push(RUN_START, { + channel.push(RUN_START, { run_id: state.activeJob, job_id: state.activeJob, - // input_dataclip_id what about this guy? }); } @@ -93,12 +93,12 @@ export function onJobStart( export function onJobComplete( channel: Channel, state: AttemptState, - jobId: string + _evt: any // TODO need to type the RTM events nicely ) { - channel.push(RUN_COMPLETE, { - run_id: state.activeJob, - job_id: state.activeJob, - // input_dataclip_id what about this guy? + channel.push(RUN_COMPLETE, { + run_id: state.activeRun!, + job_id: state.activeJob!, + // output_dataclip what about this guy? }); delete state.activeRun; @@ -106,7 +106,7 @@ export function onJobComplete( } export function onWorkflowStart(channel: Channel) { - channel.push(ATTEMPT_START); + channel.push(ATTEMPT_START); } export function onWorkflowComplete( @@ -116,7 +116,7 @@ export function onWorkflowComplete( ) { state.result = evt.state; - channel.push(ATTEMPT_COMPLETE, { + channel.push(ATTEMPT_COMPLETE, { dataclip: evt.state, }); } @@ -124,14 +124,14 @@ export function onWorkflowComplete( export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { // we basically just forward the log to lightning // but we also need to attach the log id - const evt = { + const evt: ATTEMPT_LOG_PAYLOAD = { ...log, - attempt_id: state.plan.id, + attempt_id: state.plan.id!, }; if (state.activeRun) { evt.run_id = state.activeRun; } - channel.push(ATTEMPT_LOG, evt); + channel.push(ATTEMPT_LOG, evt); } export async function prepareAttempt(channel: Channel) { diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index bc0ecfa44..a4612fac7 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -1,27 +1,65 @@ +import { JSONLog } from '@openfn/logger'; + // track socket event names as constants to keep refactoring easier export const CLAIM = 'attempt:claim'; // this is the lightning payload export type CLAIM_PAYLOAD = { demand?: number }; -export type CLAIM_REPLY_PAYLOAD = Array<{ id: string; token?: string }>; +export type CLAIM_REPLY = Array<{ id: string; token?: string }>; export const GET_ATTEMPT = 'fetch:attempt'; -export type GET_ATTEMPT_PAYLOAD = undefined; // no payload +export type GET_ATTEMPT_PAYLOAD = void; // no payload +// This is basically the attempt, which needs defining properly +export type GET_ATTEMPT_REPLY = { + id: string; + workflow: {}; + options: {}; + dataclip: string; +}; export const GET_CREDENTIAL = 'fetch:credential'; -// export type GET_CREDENTIAL_PAYLOAD = +export type GET_CREDENTIAL_PAYLOAD = { id: string }; +// credential in-line, no wrapper, arbitrary data +export type GET_CREDENTIAL_REPLY = {}; export const GET_DATACLIP = 'fetch:dataclip'; -// export type GET_DATACLIP_PAYLOAD = +export type GET_DATACLIP_PAYLOAD = { id: string }; +// dataclip in-line, no wrapper, arbitrary data +export type GET_DATACLIP_REPLY = {}; export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp +export type ATTEMPT_START_PAYLOAD = void; // no payload +export type ATTEMPT_START_REPLY = void; // no payload + export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats +export type ATTEMPT_COMPLETE_PAYLOAD = { dataclip: any; stats?: any }; // TODO dataclip -> result? output_dataclip? +export type ATTEMPT_COMPLETE_REPLY = undefined; + export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time +export type ATTEMPT_LOG_PAYLOAD = JSONLog & { + attempt_id: string; + job_id?: string; + run_id?: string; +}; +export type ATTEMPT_LOG_REPLY = void; // this should not happen - this is "could not execute" rather than "complete with errors" export const ATTEMPT_ERROR = 'attempt:error'; export const RUN_START = 'run:start'; -export const RUN_COMPLETE = 'run:complete'; +export type RUN_START_PAYLOAD = { + job_id: string; + run_id: string; + attempt_id?: string; + input_dataclip_id?: string; //hmm +}; +export type RUN_START_REPLY = void; -// TODO I'd like to create payload type for each event, so that we have a central definition +export const RUN_COMPLETE = 'run:complete'; +export type RUN_COMPLETE_PAYLOAD = { + attempt_id?: string; + job_id: string; + run_id: string; + output_dataclip?: string; //hmm +}; +export type RUN_COMPLETE_REPLY = void; diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index f0c2848e4..aaf75c883 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -8,17 +8,26 @@ import { extractAttemptId } from './util'; import createPheonixMockSocketServer, { DevSocket, PhoenixEvent, - PhoenixReply, } from './socket-server'; import { ATTEMPT_COMPLETE, + ATTEMPT_COMPLETE_PAYLOAD, + ATTEMPT_COMPLETE_REPLY, ATTEMPT_LOG, + ATTEMPT_LOG_PAYLOAD, + ATTEMPT_LOG_REPLY, CLAIM, CLAIM_PAYLOAD, - CLAIM_REPLY_PAYLOAD, + CLAIM_REPLY, GET_ATTEMPT, + GET_ATTEMPT_PAYLOAD, + GET_ATTEMPT_REPLY, GET_CREDENTIAL, + GET_CREDENTIAL_PAYLOAD, + GET_CREDENTIAL_REPLY, GET_DATACLIP, + GET_DATACLIP_PAYLOAD, + GET_DATACLIP_REPLY, } from '../../events'; import type { Server } from 'http'; @@ -44,6 +53,7 @@ const createSocketAPI = ( // pass that through to the phoenix mock const wss = createPheonixMockSocketServer({ + // @ts-ignore server typings server, state, logger: logger && createLogger('PHX', { level: 'debug' }), @@ -66,11 +76,18 @@ const createSocketAPI = ( // Right now the socket gets access to all server state // But this is just a mock - Lightning can impose more restrictions if it wishes const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { - [GET_ATTEMPT]: (ws, event) => getAttempt(state, ws, event), - [GET_CREDENTIAL]: (ws, event) => getCredential(state, ws, event), - [GET_DATACLIP]: (ws, event) => getDataclip(state, ws, event), - [ATTEMPT_LOG]: (ws, event) => handleLog(state, ws, event), - [ATTEMPT_COMPLETE]: (ws, event) => { + [GET_ATTEMPT]: (ws, event: PhoenixEvent) => + getAttempt(state, ws, event), + [GET_CREDENTIAL]: (ws, event: PhoenixEvent) => + getCredential(state, ws, event), + [GET_DATACLIP]: (ws, event: PhoenixEvent) => + getDataclip(state, ws, event), + [ATTEMPT_LOG]: (ws, event: PhoenixEvent) => + handleLog(state, ws, event), + [ATTEMPT_COMPLETE]: ( + ws, + event: PhoenixEvent + ) => { handleAttemptComplete(state, ws, event, attemptId); unsubscribe(); }, @@ -97,9 +114,9 @@ const createSocketAPI = ( let count = 1; const payload = { - status: 'ok', - response: [], - } as PhoenixReply['payload']; + status: 'ok' as const, + response: [] as CLAIM_REPLY, + }; while (count > 0 && queue.length) { // TODO assign the worker id to the attempt @@ -116,15 +133,19 @@ const createSocketAPI = ( logger?.info('No claims (queue empty)'); } - ws.reply({ ref, topic, payload }); + ws.reply({ ref, topic, payload }); } - function getAttempt(state: ServerState, ws: DevSocket, evt) { + function getAttempt( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { const { ref, topic } = evt; const attemptId = extractAttemptId(topic); - const attempt = state.attempts[attemptId]; + const attempt = state.attempts[attemptId]; /// TODO this is badly typed - ws.reply({ + ws.reply({ ref, topic, payload: { @@ -134,11 +155,15 @@ const createSocketAPI = ( }); } - function getCredential(state: ServerState, ws: DevSocket, evt) { + function getCredential( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { const { ref, topic, payload } = evt; const response = state.credentials[payload.id]; // console.log(topic, event, response); - ws.reply({ + ws.reply({ ref, topic, payload: { @@ -148,11 +173,15 @@ const createSocketAPI = ( }); } - function getDataclip(state: ServerState, ws: DevSocket, evt) { + function getDataclip( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { const { ref, topic, payload } = evt; const response = state.dataclips[payload.id]; - ws.reply({ + ws.reply({ ref, topic, payload: { @@ -162,13 +191,17 @@ const createSocketAPI = ( }); } - function handleLog(state: ServerState, ws: DevSocket, evt) { + function handleLog( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { const { ref, topic, payload } = evt; const { attempt_id: attemptId } = payload; state.pending[attemptId].logs.push(payload); - ws.reply({ + ws.reply({ ref, topic, payload: { @@ -180,7 +213,7 @@ const createSocketAPI = ( function handleAttemptComplete( state: ServerState, ws: DevSocket, - evt: PhoenixEvent, + evt: PhoenixEvent, attemptId: string ) { const { ref, topic, payload } = evt; @@ -198,7 +231,7 @@ const createSocketAPI = ( logs: state.pending[attemptId].logs, }); - ws.reply({ + ws.reply({ ref, topic, payload: { diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index a10478ba3..606d83809 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -22,7 +22,7 @@ export type DevSocket = WebSocket & { export type PhoenixEvent

= { topic: Topic; event: string; - payload?: P; + payload: P; ref: string; }; @@ -30,7 +30,7 @@ export type PhoenixReply = { topic: Topic; payload: { status: 'ok' | 'error' | 'timeout'; - response: R; + response?: R; }; ref: string; }; diff --git a/packages/rtm-server/src/mock/sockets.ts b/packages/rtm-server/src/mock/sockets.ts new file mode 100644 index 000000000..2a08cf1cc --- /dev/null +++ b/packages/rtm-server/src/mock/sockets.ts @@ -0,0 +1,55 @@ +type EventHandler = (evt?: any) => void; + +// Mock websocket implementations +export const mockChannel = (callbacks: Record = {}) => { + const c = { + on: (event: string, fn: EventHandler) => { + // TODO support multiple callbacks + callbacks[event] = fn; + }, + push:

(event: string, payload?: P) => { + // if a callback was registered, trigger it + // otherwise do nothing + let result: any; + if (callbacks[event]) { + result = callbacks[event](payload); + } + + return { + receive: (_status: string, callback: EventHandler) => { + setTimeout(() => callback(result), 1); + }, + }; + }, + join: () => { + const receive = { + receive: (status: string, callback: EventHandler) => { + if (status === 'ok') { + setTimeout(() => callback(), 1); + } + // TODO error and timeout? + return receive; + }, + }; + return receive; + }, + }; + return c; +}; + +export const mockSocket = () => { + const channels: Record> = {}; + return { + onOpen: (callback: EventHandler) => { + setTimeout(() => callback(), 1); + }, + connect: () => { + // noop + // TODO maybe it'd be helpful to throw if the channel isn't connected? + }, + channel: (topic: string) => { + channels[topic] = mockChannel(); + return channels[topic]; + }, + }; +}; diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index 2b421504c..607389c31 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -53,8 +53,20 @@ export type CancelablePromise = Promise & { cancel: () => void; }; -export type Channel = typeof phx.Channel; +type ReceiveHook = { + receive: ( + status: 'ok' | 'timeout' | 'error', + callback: (payload?: any) => void + ) => ReceiveHook; +}; + +export type Channel = { + on: (event: string, fn: (evt: any) => void) => void; + // TODO it would be super nice to infer the event from the payload + push:

(event: string, payload?: P) => ReceiveHook; + join:

(event: string, payload?: P) => ReceiveHook; +}; // type RuntimeExecutionPlanID = string; // type JobEdge = { diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index 40b14d173..bd3176ad5 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -24,8 +24,8 @@ import { loadCredential, } from '../../src/api/execute'; import createMockRTM from '../../src/mock/runtime-manager'; +import { mockChannel } from '../../src/mock/sockets'; import { attempts } from '../mock/data'; -import { mockChannel } from '../util'; test('prepareAttempt should get the attempt body', async (t) => { const attempt = attempts['attempt-1']; diff --git a/packages/rtm-server/test/api/workloop.test.ts b/packages/rtm-server/test/api/workloop.test.ts index 57fef7bb7..3f802e3f5 100644 --- a/packages/rtm-server/test/api/workloop.test.ts +++ b/packages/rtm-server/test/api/workloop.test.ts @@ -1,6 +1,8 @@ import test from 'ava'; -import { mockChannel, sleep } from '../util'; +import { sleep } from '../util'; + +import { mockChannel } from '../../src/mock/sockets'; import startWorkloop from '../../src/api/workloop'; import { CLAIM } from '../../src/events'; @@ -40,7 +42,7 @@ test('workloop sends the attempts:claim event', (t) => { }); }); -test.only('workloop sends the attempts:claim event several times ', (t) => { +test('workloop sends the attempts:claim event several times ', (t) => { return new Promise((done) => { let cancel; let count = 0; diff --git a/packages/rtm-server/test/util.test.ts b/packages/rtm-server/test/mock/sockets.test.ts similarity index 96% rename from packages/rtm-server/test/util.test.ts rename to packages/rtm-server/test/mock/sockets.test.ts index d03a8d2ad..9c7969638 100644 --- a/packages/rtm-server/test/util.test.ts +++ b/packages/rtm-server/test/mock/sockets.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { mockSocket, mockChannel } from './util'; +import { mockSocket, mockChannel } from '../../src/mock/sockets'; test('mock channel: join', (t) => { return new Promise((done) => { diff --git a/packages/rtm-server/test/util.ts b/packages/rtm-server/test/util.ts index df8c52907..33d822a42 100644 --- a/packages/rtm-server/test/util.ts +++ b/packages/rtm-server/test/util.ts @@ -1,5 +1,5 @@ export const wait = (fn, maxAttempts = 100) => - new Promise((resolve) => { + new Promise((resolve) => { let count = 0; let ival = setInterval(() => { count++; @@ -11,7 +11,7 @@ export const wait = (fn, maxAttempts = 100) => if (count == maxAttempts) { clearInterval(ival); - resolve(); + resolve(null); } }, 100); }); @@ -29,56 +29,3 @@ export const sleep = (delay = 100) => new Promise((resolve) => { setTimeout(resolve, delay); }); - -export const mockChannel = (callbacks = {}) => { - const c = { - on: (event, fn) => { - // TODO support multiple callbacks - callbacks[event] = fn; - }, - push: (event: string, payload?: any) => { - // if a callback was registered, trigger it - // otherwise do nothing - let result; - if (callbacks[event]) { - result = callbacks[event](payload); - } - - return { - receive: (status, callback) => { - setTimeout(() => callback(result), 1); - }, - }; - }, - join: () => { - const receive = { - receive: (status, callback) => { - if (status === 'ok') { - setTimeout(() => callback(), 1); - } - // TODO error and timeout? - return receive; - }, - }; - return receive; - }, - }; - return c; -}; - -export const mockSocket = () => { - const channels = {}; - return { - onOpen: (callback) => { - setTimeout(() => callback(), 1); - }, - connect: () => { - // noop - // TODO maybe it'd be helpful to throw if the channel isn't connected? - }, - channel: (topic: string) => { - channels[topic] = mockChannel(); - return channels[topic]; - }, - }; -}; From b42c79b70e1d2ab615c7b77c42fc9119b495cbbc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 27 Sep 2023 18:19:27 +0100 Subject: [PATCH 091/232] rtm: add auth token to attempt:* channel --- packages/rtm-server/src/api/connect.ts | 44 +++++++++ packages/rtm-server/src/api/execute.ts | 11 ++- packages/rtm-server/src/api/start-attempt.ts | 42 ++++++++ .../src/mock/lightning/api-sockets.ts | 5 +- .../src/mock/lightning/socket-server.ts | 15 ++- packages/rtm-server/src/mock/sockets.ts | 27 +++++- packages/rtm-server/src/server.ts | 96 +++++-------------- .../rtm-server/test/api/start-attempt.test.ts | 43 +++++++++ .../rtm-server/test/mock/lightning.test.ts | 64 +++++-------- packages/rtm-server/test/mock/sockets.test.ts | 32 +++++++ packages/rtm-server/test/server.test.ts | 6 +- 11 files changed, 261 insertions(+), 124 deletions(-) create mode 100644 packages/rtm-server/src/api/connect.ts create mode 100644 packages/rtm-server/src/api/start-attempt.ts create mode 100644 packages/rtm-server/test/api/start-attempt.test.ts diff --git a/packages/rtm-server/src/api/connect.ts b/packages/rtm-server/src/api/connect.ts new file mode 100644 index 000000000..1b873d4fe --- /dev/null +++ b/packages/rtm-server/src/api/connect.ts @@ -0,0 +1,44 @@ +import phx from 'phoenix-channels'; +import { Channel } from '../types'; + +type SocketAndChannel = { + socket: phx.Socket; + channel: Channel; +}; + +// This will open up a websocket channel to lightning +// TODO auth +export const connectToLightning = ( + endpoint: string, + _serverId: string, + Socket = phx.Socket +) => { + return new Promise((done) => { + let socket = new Socket(endpoint /*,{params: {userToken: "123"}}*/); + + // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) + // Do we infinitely try to reconnect? + // Consider what happens when the connection drops + // Unit tests on all of these behaviours! + socket.onOpen(() => { + // join the queue channel + const channel = socket.channel('attempts:queue'); + + channel + .join() + .receive('ok', () => { + done({ socket, channel }); + }) + .receive('error', (e: any) => { + console.log('ERROR', e); + }) + .receive('timeout', (e: any) => { + console.log('TIMEOUT', e); + }); + }); + + socket.connect(); + }); +}; + +export default connectToLightning; diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 4960656a8..932abef41 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -53,11 +53,12 @@ export function execute(channel: Channel, rtm, plan: ExecutionPlan) { // const context = { channel, state, logger } rtm.listen(plan.id, { - 'workflow-start': (evt) => onWorkflowStart(channel), - 'job-start': (evt) => onJobStart(channel, state, evt), - 'job-complete': (evt) => onJobComplete(channel, state, evt), - log: (evt) => onJobLog(channel, state, evt), - 'workflow-complete': (evt) => { + // TODO load event types from runtime-manager + 'workflow-start': (evt: any) => onWorkflowStart(channel), + 'job-start': (evt: any) => onJobStart(channel, state, evt), + 'job-complete': (evt: any) => onJobComplete(channel, state, evt), + log: (evt: any) => onJobLog(channel, state, evt), + 'workflow-complete': (evt: any) => { onWorkflowComplete(channel, state, evt); resolve(evt.state); }, diff --git a/packages/rtm-server/src/api/start-attempt.ts b/packages/rtm-server/src/api/start-attempt.ts new file mode 100644 index 000000000..0b0a627f8 --- /dev/null +++ b/packages/rtm-server/src/api/start-attempt.ts @@ -0,0 +1,42 @@ +import phx from 'phoenix-channels'; +import convertAttempt from '../util/convert-attempt'; +import { getWithReply } from '../util'; +import { Attempt, Channel } from '../types'; +import { ExecutionPlan } from '@openfn/runtime'; +import { GET_ATTEMPT } from '../events'; + +// TODO what happens if this channel join fails? +// Lightning could vanish, channel could error on its side, or auth could be wrong +// We don't have a good feedback mechanism yet - attempts:queue is the only channel +// we can feedback to +// Maybe we need a general errors channel +const joinAttemptChannel = ( + socket: phx.Socket, + token: string, + attemptId: string +) => { + return new Promise<{ channel: Channel; plan: ExecutionPlan }>( + (resolve, reject) => { + const channel = socket.channel(`attempt:${attemptId}`, { token }); + channel + .join() + .receive('ok', async () => { + const plan = await loadAttempt(channel); + resolve({ channel, plan }); + }) + .receive('error', (err) => { + reject(err); + }); + } + ); +}; + +export default joinAttemptChannel; + +export async function loadAttempt(channel: Channel) { + // first we get the attempt body through the socket + const attemptBody = await getWithReply(channel, GET_ATTEMPT); + // then we generate the execution plan + const plan = convertAttempt(attemptBody as Attempt); + return plan; +} diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index aaf75c883..7278a731b 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -76,6 +76,7 @@ const createSocketAPI = ( // Right now the socket gets access to all server state // But this is just a mock - Lightning can impose more restrictions if it wishes const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { + // TODO how will I validate the join in my mock? [GET_ATTEMPT]: (ws, event: PhoenixEvent) => getAttempt(state, ws, event), [GET_CREDENTIAL]: (ws, event: PhoenixEvent) => @@ -122,7 +123,9 @@ const createSocketAPI = ( // TODO assign the worker id to the attempt // Not needed by the mocks at the moment const next = queue.shift(); - payload.response.push({ id: next! }); + // TODO the token in the mock is trivial because we're not going to do any validation on it yet + // TODO need to save the token associated with this attempt + payload.response.push({ id: next!, token: 'x.y.z' }); count -= 1; startAttempt(next!); diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 606d83809..d81dc560e 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -13,6 +13,8 @@ import type { Logger } from '@openfn/logger'; type Topic = string; +export type PhoenixEventStatus = 'ok' | 'error' | 'timeout'; + // websocket with a couple of dev-friendly APIs export type DevSocket = WebSocket & { reply: (evt: PhoenixReply) => void; @@ -29,7 +31,7 @@ export type PhoenixEvent

= { export type PhoenixReply = { topic: Topic; payload: { - status: 'ok' | 'error' | 'timeout'; + status: PhoenixEventStatus; response?: R; }; ref: string; @@ -85,16 +87,21 @@ function createServer({ }); }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket - phx_join: (ws: DevSocket, { topic, ref }: PhoenixEvent) => { - let status = 'ok'; + phx_join: (ws: DevSocket, { topic, ref, payload }: PhoenixEvent) => { + let status: PhoenixEventStatus = 'ok'; let response = 'ok'; + // Validation on attempt:* channels // TODO is this logic in the right place? if (topic.startsWith(ATTEMPT_PREFIX)) { const attemptId = extractAttemptId(topic); if (!state.pending[attemptId]) { status = 'error'; - response = 'invalid_attempt'; + response = 'invalid_attempt_id'; + } else if (!payload.token) { + // TODO better token validation here + status = 'error'; + response = 'invalid_token'; } } ws.reply({ diff --git a/packages/rtm-server/src/mock/sockets.ts b/packages/rtm-server/src/mock/sockets.ts index 2a08cf1cc..c4c96f3d0 100644 --- a/packages/rtm-server/src/mock/sockets.ts +++ b/packages/rtm-server/src/mock/sockets.ts @@ -22,6 +22,21 @@ export const mockChannel = (callbacks: Record = {}) => { }; }, join: () => { + if (callbacks.join) { + // This is an attempt to mock a join fail + // not sure it works that well... + // @ts-ignore + const { status, response } = callbacks.join(); + const receive = { + receive: (requestedStatus: string, callback: EventHandler) => { + if (requestedStatus === status) { + setTimeout(() => callback(response), 1); + } + return receive; + }, + }; + return receive; + } const receive = { receive: (status: string, callback: EventHandler) => { if (status === 'ok') { @@ -37,8 +52,10 @@ export const mockChannel = (callbacks: Record = {}) => { return c; }; -export const mockSocket = () => { - const channels: Record> = {}; +type ChannelMap = Record>; + +export const mockSocket = (_endpoint: string, channels: ChannelMap) => { + const allChannels: ChannelMap = channels || {}; return { onOpen: (callback: EventHandler) => { setTimeout(() => callback(), 1); @@ -48,8 +65,10 @@ export const mockSocket = () => { // TODO maybe it'd be helpful to throw if the channel isn't connected? }, channel: (topic: string) => { - channels[topic] = mockChannel(); - return channels[topic]; + if (!allChannels[topic]) { + allChannels[topic] = mockChannel(); + } + return allChannels[topic]; }, }; }; diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index d75c22ad6..19d274d63 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -1,12 +1,14 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; -import phx from 'phoenix-channels'; + import { createMockLogger, Logger } from '@openfn/logger'; import createRestAPI from './api-rest'; import startWorkloop from './api/workloop'; -import { execute, prepareAttempt } from './api/execute'; +import { execute } from './api/execute'; +import joinAttemptChannel from './api/start-attempt'; +import connectToLightning from './api/connect'; type ServerOptions = { backoff?: number; @@ -17,41 +19,6 @@ type ServerOptions = { logger?: Logger; }; -// This will open up a websocket channel to lightning -// TODO auth -export const connectToLightning = ( - endpoint: string, - id: string, - Socket = phx.Socket -) => { - return new Promise((done) => { - let socket = new Socket(endpoint /*,{params: {userToken: "123"}}*/); - - // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) - // Do we infinitely try to reconnect? - // Consider what happens when the connection drops - // Unit tests on all of these behaviours! - socket.onOpen(() => { - // join the queue channel - const channel = socket.channel('attempts:queue'); - - channel - .join() - .receive('ok', () => { - done([socket, channel]); - }) - .receive('error', (e: any) => { - console.log('ERROR', e); - }) - .receive('timeout', (e: any) => { - console.log('TIMEOUT', e); - }); - }); - - socket.connect(); - }); -}; - function createServer(rtm: any, options: ServerOptions = {}) { const logger = options.logger || createMockLogger(); const port = options.port || 1234; @@ -77,47 +44,36 @@ function createServer(rtm: any, options: ServerOptions = {}) { logger.info('Closing server'); }; - // TODO this needs loads more unit testing - deffo need to pull it into its own funciton - const handleAttempt = async (socket, attempt) => { - const channel = socket.channel(`attempt:${attempt}`); - channel.join().receive('ok', async () => { - const plan = await prepareAttempt(channel); - console.log(plan); - execute(channel, rtm, plan); - }); + type StartAttemptArgs = { + id: string; + token: string; }; if (options.lightning) { logger.log('Starting work loop at', options.lightning); - connectToLightning(options.lightning, rtm.id).then(([socket, channel]) => { - // TODO maybe pull this logic out so we can test it? - startWorkloop(channel, (attempt) => { - handleAttempt(socket, attempt); - }); - - // debug API to run a workflow - // Used in unit tests - // Only loads in dev mode? - // @ts-ignore - app.execute = (attempt) => { - handleAttempt(socket, attempt); - }; - }); + connectToLightning(options.lightning, rtm.id).then( + ({ socket, channel }) => { + const startAttempt = async ({ id, token }: StartAttemptArgs) => { + const { channel: attemptChannel, plan } = await joinAttemptChannel( + socket, + token, + id + ); + execute(attemptChannel, rtm, plan); + }; + + // TODO maybe pull this logic out so we can test it? + startWorkloop(channel, startAttempt); + + // debug/unit test API to run a workflow + // TODO Only loads in dev mode? + (app as any).execute = startAttempt; + } + ); } else { logger.warn('No lightning URL provided'); } - // rtm.on('workflow-complete', ({ id, state }: { id: string; state: any }) => { - // logger.log(`workflow complete: `, id); - // logger.log(state); - // postResult(rtm.id, options.lightning!, id, state); - // }); - - // rtm.on('log', ({ id, messages }: { id: string; messages: any[] }) => { - // logger.log(`${id}: `, ...messages); - // postLog(rtm.id, options.lightning!, id, messages); - // }); - // TMP doing this for tests but maybe its better done externally app.on = (...args) => rtm.on(...args); app.once = (...args) => rtm.once(...args); diff --git a/packages/rtm-server/test/api/start-attempt.test.ts b/packages/rtm-server/test/api/start-attempt.test.ts new file mode 100644 index 000000000..4ae279605 --- /dev/null +++ b/packages/rtm-server/test/api/start-attempt.test.ts @@ -0,0 +1,43 @@ +import test from 'ava'; +import { mockSocket, mockChannel } from '../../src/mock/sockets'; +import joinAttemptChannel from '../../src/api/start-attempt'; +import { GET_ATTEMPT } from '../../src/events'; + +test('should join an attempt channel with a token', async (t) => { + const socket = mockSocket('www', { + 'attempt:a': mockChannel({ + // Note that the validation logic is all handled here + join: () => ({ status: 'ok' }), + [GET_ATTEMPT]: () => ({ + id: 'a', + }), + }), + }); + + const { channel, plan } = await joinAttemptChannel(socket, 'x.y.z', 'a'); + + t.truthy(channel); + t.deepEqual(plan, { id: 'a', jobs: [] }); +}); + +test('should fail to join an attempt channel with an invalid token', async (t) => { + const socket = mockSocket('www', { + 'attempt:a': mockChannel({ + // Note that the validation logic is all handled here + // We're not testing token validation, we're testing how we respond to auth fails + join: () => ({ status: 'error', response: 'invalid-token' }), + [GET_ATTEMPT]: () => ({ + id: 'a', + }), + }), + }); + + try { + // ts-ignore + await joinAttemptChannel(socket, 'x.y.z', 'a'); + } catch (e) { + // the error here is whatever is passed as the response to the receive-error event + t.is(e, 'invalid-token'); + t.pass(); + } +}); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index d8d850f97..cd1ff50bf 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -42,15 +42,19 @@ test.after(() => { const attempt1 = attempts['attempt-1']; -const join = (channelName: string): Promise => +const join = ( + channelName: string, + params: any = {} +): Promise => new Promise((done, reject) => { - const channel = client.channel(channelName, {}); + const channel = client.channel(channelName, params); channel .join() .receive('ok', () => { done(channel); }) .receive('error', (err) => { + // err will be the response message on the payload (ie, invalid_token, invalid_attempt_id etc) reject(new Error(err)); }); }); @@ -175,7 +179,7 @@ test.serial( channel.push(CLAIM).receive('ok', (response) => { t.truthy(response); t.is(response.length, 1); - t.deepEqual(response[0], { id: 'attempt-1' }); + t.deepEqual(response[0], { id: 'attempt-1', token: 'x.y.z' }); // ensure the server state has changed t.is(server.getQueueLength(), 0); @@ -187,42 +191,26 @@ test.serial( // TODO is it even worth doing this? Easier for a socket to pull one at a time? // It would also ensure better distribution if 10 workers ask at the same time, they'll get // one each then come back for more -test.serial.skip( - 'claim attempt: reply with multiple attempt ids', - (t) => - new Promise(async (done) => { - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 3); - - const channel = await join('attempts:queue'); - - // // response is an array of attempt ids - // channel.push(CLAIM, { count: 3 }).receive('ok', (response) => { - // t.truthy(response); - // t.is(response.length, 1); - // t.is(response[0], 'attempt-1'); - - // // ensure the server state has changed - // t.is(server.getQueueLength(), 0); - // done(); - // }); - }) -); - -// TODO get credentials -// TODO get state +test.todo('claim attempt: reply with multiple attempt ids'); test.serial('create a channel for an attempt', async (t) => { server.startAttempt('wibble'); - await join('attempt:wibble'); + await join('attempt:wibble', { token: 'a.b.c' }); t.pass('connection ok'); }); +test.serial('do not allow to join a channel without a token', async (t) => { + server.startAttempt('wibble'); + await t.throwsAsync(() => join('attempt:wibble'), { + message: 'invalid_token', + }); +}); + +test.todo('do not allow to join a channel without a valid token'); + test.serial('reject channels for attempts that are not started', async (t) => { await t.throwsAsync(() => join('attempt:xyz'), { - message: 'invalid_attempt', + message: 'invalid_attempt_id', }); }); @@ -231,7 +219,7 @@ test.serial('get attempt data through the attempt channel', async (t) => { server.registerAttempt(attempt1); server.startAttempt(attempt1.id); - const channel = await join(`attempt:${attempt1.id}`); + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(GET_ATTEMPT).receive('ok', (p) => { t.deepEqual(p, attempt1); done(); @@ -245,7 +233,7 @@ test.serial('complete an attempt through the attempt channel', async (t) => { server.registerAttempt(a); server.startAttempt(a.id); - const channel = await join(`attempt:${a.id}`); + const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); channel .push(ATTEMPT_COMPLETE, { dataclip: { answer: 42 } }) .receive('ok', () => { @@ -270,7 +258,7 @@ test.serial('logs are saved and acknowledged', async (t) => { time: new Date().getTime(), } as JSONLog; - const channel = await join(`attempt:${attempt1.id}`); + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(ATTEMPT_LOG, log).receive('ok', () => { const { pending } = server.getState(); const [savedLog] = pending[attempt1.id].logs; @@ -286,7 +274,7 @@ test.serial('unsubscribe after attempt complete', async (t) => { server.registerAttempt(a); server.startAttempt(a.id); - const channel = await join(`attempt:${a.id}`); + const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); channel.push(ATTEMPT_COMPLETE).receive('ok', () => { // After the complete event, the listener should unsubscribe to the channel // The mock will send an error to any unhandled events in that channel @@ -303,7 +291,7 @@ test.serial('get credential through the attempt channel', async (t) => { server.startAttempt(attempt1.id); server.addCredential('a', credentials['a']); - const channel = await join(`attempt:${attempt1.id}`); + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(GET_CREDENTIAL, { id: 'a' }).receive('ok', (result) => { t.deepEqual(result, credentials['a']); done(); @@ -316,7 +304,7 @@ test.serial('get dataclip through the attempt channel', async (t) => { server.startAttempt(attempt1.id); server.addDataclip('d', dataclips['d']); - const channel = await join(`attempt:${attempt1.id}`); + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(GET_DATACLIP, { id: 'd' }).receive('ok', (result) => { t.deepEqual(result, dataclips['d']); done(); @@ -343,7 +331,7 @@ test.serial( done(); }); - const channel = await join(`attempt:${attempt1.id}`); + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(ATTEMPT_COMPLETE, { dataclip: result }); }); } diff --git a/packages/rtm-server/test/mock/sockets.test.ts b/packages/rtm-server/test/mock/sockets.test.ts index 9c7969638..994f2c50c 100644 --- a/packages/rtm-server/test/mock/sockets.test.ts +++ b/packages/rtm-server/test/mock/sockets.test.ts @@ -14,6 +14,38 @@ test('mock channel: join', (t) => { }); }); +test('mock channel: join with mock handler', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + join: () => ({ status: 'ok' }), + }); + + t.assert(channel.hasOwnProperty('push')); + t.assert(channel.hasOwnProperty('join')); + + channel.join().receive('ok', () => { + t.pass(); + done(); + }); + }); +}); + +test('mock channel: error on join', (t) => { + return new Promise((done) => { + const channel = mockChannel({ + join: () => ({ status: 'error', response: 'fail' }), + }); + + t.assert(channel.hasOwnProperty('push')); + t.assert(channel.hasOwnProperty('join')); + + channel.join().receive('error', () => { + t.pass(); + done(); + }); + }); +}); + test('mock channel: should invoke handler with payload', (t) => { return new Promise((done) => { const channel = mockChannel({ diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index 70a908ccd..ecdc3a4db 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -1,9 +1,11 @@ import test from 'ava'; import WebSocket, { WebSocketServer } from 'ws'; -import createServer, { connectToLightning } from '../src/server'; +import createServer from '../src/server'; +import connectToLightning from '../src/api/connect'; import createMockRTM from '../src/mock/runtime-manager'; -import { mockChannel, mockSocket, sleep } from './util'; +import { sleep } from './util'; +import { mockChannel, mockSocket } from '../src/mock/sockets'; import { CLAIM } from '../src/events'; // Unit tests against the RTM web server From de6a544fed6efefa553dd0e85dd2382a67f62b40 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 14:50:24 +0100 Subject: [PATCH 092/232] rtm: move some tests around I think. May have got this wrong. --- packages/rtm-server/src/api/execute.ts | 16 +------- packages/rtm-server/test/api/execute.test.ts | 39 ------------------- .../rtm-server/test/api/start-attempt.test.ts | 38 ++++++++++++++++++ 3 files changed, 39 insertions(+), 54 deletions(-) diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 932abef41..086e2e999 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -6,7 +6,7 @@ // it isn't an actual worker, but a BRIDGE between a worker and lightning import crypto from 'node:crypto'; import { JSONLog } from '@openfn/logger'; -import convertAttempt from '../util/convert-attempt'; + // this managers the worker //i would like functions to be testable, and I'd like the logic to be readable @@ -135,20 +135,6 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { channel.push(ATTEMPT_LOG, evt); } -export async function prepareAttempt(channel: Channel) { - // first we get the attempt body through the socket - const attemptBody = (await getWithReply(channel, GET_ATTEMPT)) as Attempt; - - // then we generate the execution plan - const plan = convertAttempt(attemptBody); - - return plan; - // difficulty: we need to tell the rtm how to callback for - // credentials and state (which should both be lazy and part of the run) - // I guess this is generic - given an attempt id I can lookup the channel and return this information - // then we call the excute function. Or return the promise and let someone else do that -} - export async function loadState(channel: Channel, stateId: string) { return getWithReply(channel, GET_DATACLIP, { dataclip_id: stateId }); } diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index bd3176ad5..7a97ee054 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -2,7 +2,6 @@ import test from 'ava'; import { JSONLog } from '@openfn/logger'; import { - GET_ATTEMPT, RUN_START, RUN_COMPLETE, ATTEMPT_LOG, @@ -12,7 +11,6 @@ import { GET_DATACLIP, } from '../../src/events'; import { - prepareAttempt, onJobStart, onJobComplete, onJobLog, @@ -25,43 +23,6 @@ import { } from '../../src/api/execute'; import createMockRTM from '../../src/mock/runtime-manager'; import { mockChannel } from '../../src/mock/sockets'; -import { attempts } from '../mock/data'; - -test('prepareAttempt should get the attempt body', async (t) => { - const attempt = attempts['attempt-1']; - let didCallGetAttempt = false; - const channel = mockChannel({ - [GET_ATTEMPT]: () => { - // TODO should be no payload (or empty payload) - didCallGetAttempt = true; - return attempt; - }, - }); - - await prepareAttempt(channel, 'a1'); - t.true(didCallGetAttempt); -}); - -test('prepareAttempt should return an execution plan', async (t) => { - const attempt = attempts['attempt-1']; - - const channel = mockChannel({ - [GET_ATTEMPT]: () => attempt, - }); - - const plan = await prepareAttempt(channel, 'a1'); - t.deepEqual(plan, { - id: 'attempt-1', - jobs: [ - { - id: 'trigger', - configuration: 'a', - expression: 'fn(a => a)', - adaptor: '@openfn/language-common@1.0.0', - }, - ], - }); -}); test('jobStart should set a run id and active job on state', async (t) => { const plan = { id: 'attempt-1' }; diff --git a/packages/rtm-server/test/api/start-attempt.test.ts b/packages/rtm-server/test/api/start-attempt.test.ts index 4ae279605..51c447a60 100644 --- a/packages/rtm-server/test/api/start-attempt.test.ts +++ b/packages/rtm-server/test/api/start-attempt.test.ts @@ -2,6 +2,44 @@ import test from 'ava'; import { mockSocket, mockChannel } from '../../src/mock/sockets'; import joinAttemptChannel from '../../src/api/start-attempt'; import { GET_ATTEMPT } from '../../src/events'; +import { loadAttempt } from '../../src/api/start-attempt'; +import { attempts } from '../mock/data'; + +test('loadAttempt should get the attempt body', async (t) => { + const attempt = attempts['attempt-1']; + let didCallGetAttempt = false; + const channel = mockChannel({ + [GET_ATTEMPT]: () => { + // TODO should be no payload (or empty payload) + didCallGetAttempt = true; + return attempt; + }, + }); + + await loadAttempt(channel); + t.true(didCallGetAttempt); +}); + +test('loadAttempt should return an execution plan', async (t) => { + const attempt = attempts['attempt-1']; + + const channel = mockChannel({ + [GET_ATTEMPT]: () => attempt, + }); + + const plan = await loadAttempt(channel); + t.deepEqual(plan, { + id: 'attempt-1', + jobs: [ + { + id: 'trigger', + configuration: 'a', + expression: 'fn(a => a)', + adaptor: '@openfn/language-common@1.0.0', + }, + ], + }); +}); test('should join an attempt channel with a token', async (t) => { const socket = mockSocket('www', { From cc1aa1d7fc34d515ac6c7f239dd37d9cd75d6c52 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 15:17:20 +0100 Subject: [PATCH 093/232] rtm: add support for worker token Now we send a token up to lightning on connect --- packages/rtm-server/package.json | 1 + packages/rtm-server/src/api/connect.ts | 26 ++++--- .../src/mock/lightning/socket-server.ts | 1 + packages/rtm-server/src/mock/sockets.ts | 22 ++++-- packages/rtm-server/src/server.ts | 4 +- packages/rtm-server/src/start.ts | 12 ++++ packages/rtm-server/src/types.d.ts | 6 ++ packages/rtm-server/src/util/worker-token.ts | 23 +++++++ packages/rtm-server/test/api/connect.test.ts | 68 +++++++++++++++++++ packages/rtm-server/test/mock/sockets.test.ts | 47 ++++++++++++- packages/rtm-server/test/server.test.ts | 11 +-- .../rtm-server/test/util/worker-token.test.ts | 42 ++++++++++++ 12 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 packages/rtm-server/src/util/worker-token.ts create mode 100644 packages/rtm-server/test/api/connect.test.ts create mode 100644 packages/rtm-server/test/util/worker-token.test.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 5c3903286..4b0394d56 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -23,6 +23,7 @@ "@openfn/runtime-manager": "workspace:*", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", + "jose": "^4.14.6", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", "koa-logger": "^3.2.1", diff --git a/packages/rtm-server/src/api/connect.ts b/packages/rtm-server/src/api/connect.ts index 1b873d4fe..f029f89eb 100644 --- a/packages/rtm-server/src/api/connect.ts +++ b/packages/rtm-server/src/api/connect.ts @@ -1,20 +1,24 @@ import phx from 'phoenix-channels'; -import { Channel } from '../types'; +import generateWorkerToken from '../util/worker-token'; +import type { Socket, Channel } from '../types'; type SocketAndChannel = { - socket: phx.Socket; + socket: Socket; channel: Channel; }; -// This will open up a websocket channel to lightning -// TODO auth export const connectToLightning = ( endpoint: string, - _serverId: string, - Socket = phx.Socket + serverId: string, + secret: string, + SocketConstructor: Socket = phx.Socket ) => { - return new Promise((done) => { - let socket = new Socket(endpoint /*,{params: {userToken: "123"}}*/); + return new Promise(async (done, reject) => { + // TODO does this token need to be fed back anyhow? + // I think it's just used to connect and then forgotten? + // If we reconnect we need a new token I guess? + const token = await generateWorkerToken(secret, serverId); + const socket = new SocketConstructor(endpoint, { params: { token } }); // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) // Do we infinitely try to reconnect? @@ -22,6 +26,7 @@ export const connectToLightning = ( // Unit tests on all of these behaviours! socket.onOpen(() => { // join the queue channel + // TODO should this send the worker token? const channel = socket.channel('attempts:queue'); channel @@ -37,6 +42,11 @@ export const connectToLightning = ( }); }); + // TODO what even happens if the connection fails? + socket.onError((e) => { + reject(e); + }); + socket.connect(); }); }; diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index d81dc560e..0ae2d0386 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -112,6 +112,7 @@ function createServer({ }, }; + // TODO need to verify jwt here wsServer.on('connection', function (ws: DevSocket, _req: any) { logger?.info('new client connected'); diff --git a/packages/rtm-server/src/mock/sockets.ts b/packages/rtm-server/src/mock/sockets.ts index c4c96f3d0..fc1db1081 100644 --- a/packages/rtm-server/src/mock/sockets.ts +++ b/packages/rtm-server/src/mock/sockets.ts @@ -54,15 +54,29 @@ export const mockChannel = (callbacks: Record = {}) => { type ChannelMap = Record>; -export const mockSocket = (_endpoint: string, channels: ChannelMap) => { +export const mockSocket = ( + _endpoint?: string, + channels?: ChannelMap, + connect: () => Promise = async () => {} +) => { const allChannels: ChannelMap = channels || {}; + + const callbacks: Record = {}; return { onOpen: (callback: EventHandler) => { - setTimeout(() => callback(), 1); + callbacks.onOpen = callback; + }, + onError: (callback: EventHandler) => { + callbacks.onError = callback; }, connect: () => { - // noop - // TODO maybe it'd be helpful to throw if the channel isn't connected? + connect() + .then(() => { + setTimeout(() => callbacks?.onOpen?.(), 1); + }) + .catch((e) => { + setTimeout(() => callbacks?.onError?.(e), 1); + }); }, channel: (topic: string) => { if (!allChannels[topic]) { diff --git a/packages/rtm-server/src/server.ts b/packages/rtm-server/src/server.ts index 19d274d63..cc9dcf967 100644 --- a/packages/rtm-server/src/server.ts +++ b/packages/rtm-server/src/server.ts @@ -17,6 +17,8 @@ type ServerOptions = { lightning?: string; // url to lightning instance rtm?: any; logger?: Logger; + + secret: string; // worker secret }; function createServer(rtm: any, options: ServerOptions = {}) { @@ -51,7 +53,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { if (options.lightning) { logger.log('Starting work loop at', options.lightning); - connectToLightning(options.lightning, rtm.id).then( + connectToLightning(options.lightning, rtm.id, options.secret).then( ({ socket, channel }) => { const startAttempt = async ({ id, token }: StartAttemptArgs) => { const { channel: attemptChannel, plan } = await joinAttemptChannel( diff --git a/packages/rtm-server/src/start.ts b/packages/rtm-server/src/start.ts index 3d72ae27c..8ad398aee 100644 --- a/packages/rtm-server/src/start.ts +++ b/packages/rtm-server/src/start.ts @@ -12,6 +12,7 @@ type Args = { port?: number; lightning?: string; repoDir?: string; + secret?: string; }; const logger = createLogger('SRV', { level: 'info' }); @@ -33,10 +34,20 @@ const args = yargs(hideBin(process.argv)) alias: 'd', description: 'Path to the runtime repo (where modules will be installed)', }) + .option('secret', { + alias: 's', + description: 'Worker secret (comes from WORKER_SECRET by default)', + }) .parse() as Args; if (args.lightning === 'mock') { args.lightning = 'ws://localhost:8888/api'; +} else if (!args.secret) { + if (!process.env.WORKER_SECRET) { + console.error('WORKER_SECRET is not set'); + process.exit(1); + } + args.secret = process.env.WORKER_SECRET; } // TODO the rtm needs to take callbacks to load credential, and load state @@ -56,4 +67,5 @@ createRTMServer(rtm, { port: args.port, lightning: args.lightning, logger, + secret: args.secret, }); diff --git a/packages/rtm-server/src/types.d.ts b/packages/rtm-server/src/types.d.ts index 607389c31..7f273a897 100644 --- a/packages/rtm-server/src/types.d.ts +++ b/packages/rtm-server/src/types.d.ts @@ -60,6 +60,12 @@ type ReceiveHook = { ) => ReceiveHook; }; +export declare class Socket { + constructor(endpoint: string, options: { params: any }); + onOpen(callback: () => void): void; + connect(): void; +} + export type Channel = { on: (event: string, fn: (evt: any) => void) => void; diff --git a/packages/rtm-server/src/util/worker-token.ts b/packages/rtm-server/src/util/worker-token.ts new file mode 100644 index 000000000..fe13e6ca6 --- /dev/null +++ b/packages/rtm-server/src/util/worker-token.ts @@ -0,0 +1,23 @@ +import * as jose from 'jose'; + +const alg = 'HS256'; + +const generateWorkerToken = async (secret: string, workerId: string) => { + const encodedSecret = new TextEncoder().encode(secret); + + const claims = { + worker_id: workerId, + }; + + const jwt = await new jose.SignJWT(claims) + .setProtectedHeader({ alg }) + .setIssuedAt() + .setIssuer('urn:example:issuer') + .setAudience('urn:example:audience') + // .setExpirationTime('2h') + .sign(encodedSecret); + + return jwt; +}; + +export default generateWorkerToken; diff --git a/packages/rtm-server/test/api/connect.test.ts b/packages/rtm-server/test/api/connect.test.ts new file mode 100644 index 000000000..3c11957eb --- /dev/null +++ b/packages/rtm-server/test/api/connect.test.ts @@ -0,0 +1,68 @@ +import test from 'ava'; +import * as jose from 'jose'; +import connect from '../../src/api/connect'; +import { mockSocket } from '../../src/mock/sockets'; + +test('should connect', async (t) => { + const { socket, channel } = await connect('www', 'a', 'secret', mockSocket); + + t.truthy(socket); + t.truthy(socket.connect); + t.truthy(channel); + t.truthy(channel.join); +}); + +test('should connect with an auth token', async (t) => { + const workerId = 'x'; + const secret = 'xyz'; + const encodedSecret = new TextEncoder().encode(secret); + + function createSocket(endpoint, options) { + const socket = mockSocket(endpoint, {}, async () => { + const { token } = options.params; + + const { payload } = await jose.jwtVerify(token, encodedSecret); + t.is(payload.worker_id, workerId); + }); + + return socket; + } + const { socket, channel } = await connect( + 'www', + workerId, + secret, + createSocket + ); + + t.truthy(socket); + t.truthy(socket.connect); + t.truthy(channel); + t.truthy(channel.join); +}); + +test('should fail to connect with an invalid auth token', async (t) => { + const workerId = 'x'; + const secret = 'xyz'; + const encodedSecret = new TextEncoder().encode(secret); + + function createSocket(endpoint, options) { + const socket = mockSocket(endpoint, {}, async () => { + const { token } = options.params; + + try { + await jose.jwtVerify(token, encodedSecret); + } catch (_e) { + throw new Error('auth_fail'); + } + }); + + return socket; + } + + await t.throwsAsync(connect('www', workerId, 'wrong-secret!', createSocket), { + message: 'auth_fail', + }); +}); + +// TODO maybe? +test.todo('should reconnect with backoff when connection is dropped'); diff --git a/packages/rtm-server/test/mock/sockets.test.ts b/packages/rtm-server/test/mock/sockets.test.ts index 994f2c50c..46ad6aa4c 100644 --- a/packages/rtm-server/test/mock/sockets.test.ts +++ b/packages/rtm-server/test/mock/sockets.test.ts @@ -93,13 +93,58 @@ test('mock socket: connect', (t) => { return new Promise((done) => { const socket = mockSocket(); - // this is a noop socket.connect(); t.pass('connected'); done(); }); }); +test('mock socket: connect and call onOpen', (t) => { + return new Promise((done) => { + const socket = mockSocket(); + + socket.onOpen(() => { + t.pass('called on open'); + done(); + }); + + socket.connect(); + }); +}); + +test('mock socket: call onOpen with customConnect', (t) => { + return new Promise((done) => { + let didCallConnect = false; + + const socket = mockSocket('www', {}, async () => { + didCallConnect = true; + }); + + socket.onOpen(() => { + t.true(didCallConnect); + done(); + }); + + socket.connect(); + }); +}); + +test('mock socket: call onError if connect throws', (t) => { + return new Promise((done) => { + const socket = mockSocket('www', {}, async () => { + throw 'err'; + }); + + socket.onError((e) => { + t.is(e, 'err'); + t.pass(); + done(); + }); + + socket.connect(); + }); +}); + test('mock socket: connect to channel', (t) => { return new Promise((done) => { const socket = mockSocket(); diff --git a/packages/rtm-server/test/server.test.ts b/packages/rtm-server/test/server.test.ts index ecdc3a4db..5917aeb92 100644 --- a/packages/rtm-server/test/server.test.ts +++ b/packages/rtm-server/test/server.test.ts @@ -34,13 +34,14 @@ test.skip('healthcheck', async (t) => { t.is(body, 'OK'); }); +// TODO this isn't testing anything now, see test/api/connect.test.ts // Not a very thorough test -test.only('connects to lightning', async (t) => { - await connectToLightning('www', 'rtm', mockSocket); - t.pass(); +// test.only('connects to lightning', async (t) => { +// await connectToLightning('www', 'rtm', mockSocket); +// t.pass(); - // TODO connections to the same socket.channel should share listners, so I think I can test the channel -}); +// // TODO connections to the same socket.channel should share listners, so I think I can test the channel +// }); // test('connects to websocket', (t) => { // let didSayHello; diff --git a/packages/rtm-server/test/util/worker-token.test.ts b/packages/rtm-server/test/util/worker-token.test.ts new file mode 100644 index 000000000..2be646b32 --- /dev/null +++ b/packages/rtm-server/test/util/worker-token.test.ts @@ -0,0 +1,42 @@ +import test from 'ava'; +import * as jose from 'jose'; + +import generateWorkerToken from '../../src/util/worker-token'; + +test('should generate a worker token as a JWT', async (t) => { + const jwt = await generateWorkerToken('abc', 'x'); + + t.truthy(jwt); + t.assert(typeof jwt === 'string'); + + const parts = jwt.split('.'); + t.is(parts.length, 3); +}); + +test('should verify the signature', async (t) => { + const secret = 'abc'; + const encodedSecret = new TextEncoder().encode(secret); + + const jwt = await generateWorkerToken(secret, 'x'); + + const { payload, protectedHeader } = await jose.jwtVerify(jwt, encodedSecret); + + t.is(payload.worker_id, 'x'); + t.deepEqual(protectedHeader, { alg: 'HS256' }); +}); + +test('should throw on verify if the signature is wrong', async (t) => { + const secret = 'abc'; + const encodedSecret = new TextEncoder().encode('xyz'); + + const jwt = await generateWorkerToken(secret, 'x'); + + await t.throwsAsync(() => jose.jwtVerify(jwt, encodedSecret)); +}); + +test('should include the server id in the payload', async (t) => { + const jwt = await generateWorkerToken('abc', 'x'); + const claims = jose.decodeJwt(jwt); + + t.is(claims.worker_id, 'x'); +}); From 3aabe65858bca71c5df34022cfb03134bc735f0f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 15:40:38 +0100 Subject: [PATCH 094/232] rtm: lightning mock requires a token --- packages/rtm-server/package.json | 1 + .../src/mock/lightning/socket-server.ts | 20 +++++++++-- .../rtm-server/test/mock/lightning.test.ts | 20 +++++++++-- pnpm-lock.yaml | 34 +++++++++++++++++++ 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 4b0394d56..41e12b5d5 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -40,6 +40,7 @@ "koa-route": "^3.2.0", "koa-websocket": "^7.0.0", "nodemon": "3.0.1", + "query-string": "^8.1.0", "ts-node": "^10.9.1", "tslib": "^2.4.0", "tsup": "^6.2.3", diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 0ae2d0386..56883e9bf 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -5,6 +5,7 @@ * It also adds some dev and debug APIs, useful for unit testing */ import { WebSocketServer, WebSocket } from 'ws'; +import querystring from 'query-string'; import { ATTEMPT_PREFIX, extractAttemptId } from './util'; import { ServerState } from './server'; @@ -112,10 +113,25 @@ function createServer({ }, }; - // TODO need to verify jwt here - wsServer.on('connection', function (ws: DevSocket, _req: any) { + wsServer.on('connection', function (ws: DevSocket, req: any) { logger?.info('new client connected'); + // Ensure that a JWT token is added to the + const [_path, query] = req.url.split('?'); + const { token } = querystring.parse(query); + + // TODO for now, there's no validation on the token in this mock + + // If there is no token (or later, if invalid), close the connection immediately + if (!token) { + logger?.error('INVALID TOKEN'); + ws.close(); + + // TODO I'd love to send back a 403 here, not sure how to do it + // (and it's not important in the mock really) + return; + } + ws.reply = ({ ref, topic, payload }: PhoenixReply) => { logger?.debug( `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index cd1ff50bf..7e301aa64 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -26,9 +26,11 @@ test.before( new Promise((done) => { server = createLightningServer({ port: 7777 }); - client = new phx.Socket(endpoint, { timeout: 50 }); - client.connect(); + // Note that we need a token to connect, but the mock here + // doesn't (yet) do any validation on that token + client = new phx.Socket(endpoint, { params: { token: 'xyz' } }); client.onOpen(done); + client.connect(); }) ); @@ -113,7 +115,19 @@ test.serial('provide a phoenix websocket at /api', (t) => { t.is(client.connectionState(), 'open'); }); -test.serial('respond to connection join requests', (t) => { +test.serial('reject ws connections without a token', (t) => { + return new Promise((done) => { + // client should be connected before this test runs + const socket = new phx.Socket(endpoint); + socket.onClose(() => { + t.pass(); + done(); + }); + socket.connect(); + }); +}); + +test.serial('respond to channel join requests', (t) => { return new Promise(async (done, reject) => { const channel = client.channel('x', {}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67c24dcfb..78b73178a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -387,6 +387,9 @@ importers: '@types/ws': specifier: ^8.5.6 version: 8.5.6 + jose: + specifier: ^4.14.6 + version: 4.14.6 koa: specifier: ^2.13.4 version: 2.13.4 @@ -430,6 +433,9 @@ importers: nodemon: specifier: 3.0.1 version: 3.0.1 + query-string: + specifier: ^8.1.0 + version: 8.1.0 ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@18.15.3)(typescript@4.6.4) @@ -2981,6 +2987,11 @@ packages: engines: {node: '>=0.10'} dev: true + /decode-uri-component@0.4.1: + resolution: {integrity: sha512-+8VxcR21HhTy8nOt6jf20w0c9CADrw1O8d+VZ/YzzCt4bJ3uBjw+D1q2osAB8RnpwwaeYBxy0HyKQxD5JBMuuQ==} + engines: {node: '>=14.16'} + dev: true + /deep-equal@1.0.1: resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} dev: false @@ -3994,6 +4005,11 @@ packages: dependencies: to-regex-range: 5.0.1 + /filter-obj@5.1.0: + resolution: {integrity: sha512-qWeTREPoT7I0bifpPUXtxkZJ1XJzxWtfoWWkdVGqa+eCr3SHW/Ocp89o8vLvbUuQnadybJpjOKu4V+RwO6sGng==} + engines: {node: '>=14.16'} + dev: true + /finalhandler@1.1.2: resolution: {integrity: sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==} engines: {node: '>= 0.8'} @@ -4911,6 +4927,10 @@ packages: '@pkgjs/parseargs': 0.11.0 dev: true + /jose@4.14.6: + resolution: {integrity: sha512-EqJPEUlZD0/CSUMubKtMaYUOtWe91tZXTWMJZoKSbLk+KtdhNdcvppH8lA9XwVu2V4Ailvsj0GBZJ2ZwDjfesQ==} + dev: false + /joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} @@ -6345,6 +6365,15 @@ packages: side-channel: 1.0.4 dev: false + /query-string@8.1.0: + resolution: {integrity: sha512-BFQeWxJOZxZGix7y+SByG3F36dA0AbTy9o6pSmKFcFz7DAj0re9Frkty3saBn3nHo3D0oZJ/+rx3r8H8r8Jbpw==} + engines: {node: '>=14.16'} + dependencies: + decode-uri-component: 0.4.1 + filter-obj: 5.1.0 + split-on-first: 3.0.0 + dev: true + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6944,6 +6973,11 @@ packages: resolution: {integrity: sha512-rr+VVSXtRhO4OHbXUiAF7xW3Bo9DuuF6C5jH+q/x15j2jniycgKbxU09Hr0WqlSLUs4i4ltHGXqTe7VHclYWyA==} dev: true + /split-on-first@3.0.0: + resolution: {integrity: sha512-qxQJTx2ryR0Dw0ITYyekNQWpz6f8dGd7vffGNflQQ3Iqj9NJ6qiZ7ELpZsJ/QBhIVAiDfXdag3+Gp8RvWa62AA==} + engines: {node: '>=12'} + dev: true + /split-string@3.1.0: resolution: {integrity: sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==} engines: {node: '>=0.10.0'} From 5dd5b156b0c065daeb8cdae48e38da9483d4ef2c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 15:52:10 +0100 Subject: [PATCH 095/232] rtm: update tests --- packages/rtm-server/test/mock/lightning.test.ts | 2 +- packages/rtm-server/test/mock/socket-server.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 7e301aa64..0ff2629e5 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -28,7 +28,7 @@ test.before( // Note that we need a token to connect, but the mock here // doesn't (yet) do any validation on that token - client = new phx.Socket(endpoint, { params: { token: 'xyz' } }); + client = new phx.Socket(endpoint, { params: { token: 'x.y.z' } }); client.onOpen(done); client.connect(); }) diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/rtm-server/test/mock/socket-server.test.ts index 9b3ecd3b5..62142b5e8 100644 --- a/packages/rtm-server/test/mock/socket-server.test.ts +++ b/packages/rtm-server/test/mock/socket-server.test.ts @@ -17,7 +17,9 @@ test.beforeEach(() => { // @ts-ignore I don't care about missing server options here server = createServer({ onMessage: (evt) => messages.push(evt) }); - socket = new phx.Socket('ws://localhost:8080'); + socket = new phx.Socket('ws://localhost:8080', { + params: { token: 'x.y.z' }, + }); socket.connect(); }); From 379e6d18c1aa2c5a4dc187f68126788468615c50 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 15:59:17 +0100 Subject: [PATCH 096/232] rtm: RUN_COMPLETE strigifies the dataclip --- packages/rtm-server/src/api/execute.ts | 4 ++-- packages/rtm-server/src/events.ts | 2 +- packages/rtm-server/test/api/execute.test.ts | 8 +++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 086e2e999..90e60f92a 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -94,12 +94,12 @@ export function onJobStart( export function onJobComplete( channel: Channel, state: AttemptState, - _evt: any // TODO need to type the RTM events nicely + evt: any // TODO need to type the RTM events nicely ) { channel.push(RUN_COMPLETE, { run_id: state.activeRun!, job_id: state.activeJob!, - // output_dataclip what about this guy? + output_dataclip: JSON.stringify(evt.state), }); delete state.activeRun; diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index a4612fac7..00a5e5899 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -60,6 +60,6 @@ export type RUN_COMPLETE_PAYLOAD = { attempt_id?: string; job_id: string; run_id: string; - output_dataclip?: string; //hmm + output_dataclip?: string; }; export type RUN_COMPLETE_REPLY = void; diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index 7a97ee054..3b902c7b3 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -62,7 +62,7 @@ test('jobStart should send a run:start event', async (t) => { }); }); -test('jobEnd should clear the run id and active job on state', async (t) => { +test('jobComplete should clear the run id and active job on state', async (t) => { const plan = { id: 'attempt-1' }; const jobId = 'job-1'; @@ -74,7 +74,7 @@ test('jobEnd should clear the run id and active job on state', async (t) => { const channel = mockChannel({}); - onJobComplete(channel, state, jobId); + onJobComplete(channel, state, { state: { x: 10 } }); t.falsy(state.activeJob); t.falsy(state.activeRun); @@ -84,6 +84,7 @@ test('jobComplete should send a run:complete event', async (t) => { return new Promise((done) => { const plan = { id: 'attempt-1' }; const jobId = 'job-1'; + const result = { x: 10 }; const state = { plan, @@ -95,12 +96,13 @@ test('jobComplete should send a run:complete event', async (t) => { [RUN_COMPLETE]: (evt) => { t.is(evt.job_id, jobId); t.truthy(evt.run_id); + t.is(evt.output_dataclip, JSON.stringify(result)); done(); }, }); - onJobComplete(channel, state, jobId); + onJobComplete(channel, state, { state: result }); }); }); From 68f47eb880d44546d5044848a7862711c34d83c5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 16:57:52 +0100 Subject: [PATCH 097/232] rtm: send dataclips as arraybuffers --- packages/rtm-server/package.json | 1 + packages/rtm-server/src/api/execute.ts | 8 ++++++- packages/rtm-server/src/events.ts | 5 +++- .../src/mock/lightning/api-sockets.ts | 12 +++++++--- .../src/mock/lightning/socket-server.ts | 15 +++++------- .../rtm-server/src/util/get-with-reply.ts | 4 ++-- packages/rtm-server/src/util/index.ts | 3 ++- packages/rtm-server/src/util/stringify.ts | 9 +++++++ packages/rtm-server/test/api/execute.test.ts | 9 +++++-- .../rtm-server/test/mock/lightning.test.ts | 10 +++++--- .../rtm-server/test/util/stringify.test.ts | 24 +++++++++++++++++++ 11 files changed, 78 insertions(+), 22 deletions(-) create mode 100644 packages/rtm-server/src/util/stringify.ts create mode 100644 packages/rtm-server/test/util/stringify.test.ts diff --git a/packages/rtm-server/package.json b/packages/rtm-server/package.json index 41e12b5d5..5663c4595 100644 --- a/packages/rtm-server/package.json +++ b/packages/rtm-server/package.json @@ -23,6 +23,7 @@ "@openfn/runtime-manager": "workspace:*", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", + "fast-safe-stringify": "^2.1.1", "jose": "^4.14.6", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", diff --git a/packages/rtm-server/src/api/execute.ts b/packages/rtm-server/src/api/execute.ts index 90e60f92a..f42f03c8b 100644 --- a/packages/rtm-server/src/api/execute.ts +++ b/packages/rtm-server/src/api/execute.ts @@ -29,6 +29,8 @@ import { Attempt, Channel } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; import { getWithReply } from '../util'; +const enc = new TextDecoder('utf-8'); + export type AttemptState = { activeRun?: string; activeJob?: string; @@ -136,7 +138,11 @@ export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { } export async function loadState(channel: Channel, stateId: string) { - return getWithReply(channel, GET_DATACLIP, { dataclip_id: stateId }); + const result = await getWithReply(channel, GET_DATACLIP, { + dataclip_id: stateId, + }); + const str = enc.decode(new Uint8Array(result)); + return JSON.parse(str); } export async function loadCredential(channel: Channel, credentialId: string) { diff --git a/packages/rtm-server/src/events.ts b/packages/rtm-server/src/events.ts index 00a5e5899..1d9f038dc 100644 --- a/packages/rtm-server/src/events.ts +++ b/packages/rtm-server/src/events.ts @@ -10,7 +10,10 @@ export type CLAIM_REPLY = Array<{ id: string; token?: string }>; export const GET_ATTEMPT = 'fetch:attempt'; export type GET_ATTEMPT_PAYLOAD = void; // no payload // This is basically the attempt, which needs defining properly -export type GET_ATTEMPT_REPLY = { +export type GET_ATTEMPT_REPLY = Uint8Array; // represents a json string Attempt + +// TODO +type Attempt = { id: string; workflow: {}; options: {}; diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/rtm-server/src/mock/lightning/api-sockets.ts index 7278a731b..a4f0904ce 100644 --- a/packages/rtm-server/src/mock/lightning/api-sockets.ts +++ b/packages/rtm-server/src/mock/lightning/api-sockets.ts @@ -31,6 +31,9 @@ import { } from '../../events'; import type { Server } from 'http'; +import { stringify } from '../../util'; + +const enc = new TextEncoder(); // this new API is websocket based // Events map to handlers @@ -146,14 +149,14 @@ const createSocketAPI = ( ) { const { ref, topic } = evt; const attemptId = extractAttemptId(topic); - const attempt = state.attempts[attemptId]; /// TODO this is badly typed + const response = state.attempts[attemptId]; /// TODO this is badly typed ws.reply({ ref, topic, payload: { status: 'ok', - response: attempt, + response, }, }); } @@ -182,7 +185,10 @@ const createSocketAPI = ( evt: PhoenixEvent ) { const { ref, topic, payload } = evt; - const response = state.dataclips[payload.id]; + const dataclip = state.dataclips[payload.id]; + + // Send the data as an ArrayBuffer (our stringify function will do this) + const response = enc.encode(stringify(dataclip)); ws.reply({ ref, diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/rtm-server/src/mock/lightning/socket-server.ts index 56883e9bf..d15f4e27d 100644 --- a/packages/rtm-server/src/mock/lightning/socket-server.ts +++ b/packages/rtm-server/src/mock/lightning/socket-server.ts @@ -9,6 +9,7 @@ import querystring from 'query-string'; import { ATTEMPT_PREFIX, extractAttemptId } from './util'; import { ServerState } from './server'; +import { stringify } from '../../util'; import type { Logger } from '@openfn/logger'; @@ -133,11 +134,9 @@ function createServer({ } ws.reply = ({ ref, topic, payload }: PhoenixReply) => { - logger?.debug( - `<< [${topic}] chan_reply_${ref} ` + JSON.stringify(payload) - ); + logger?.debug(`<< [${topic}] chan_reply_${ref} ` + stringify(payload)); ws.send( - JSON.stringify({ + stringify({ event: `chan_reply_${ref}`, ref, topic, @@ -147,9 +146,9 @@ function createServer({ }; ws.sendJSON = ({ event, ref, topic, payload }: PhoenixEvent) => { - logger?.debug(`<< [${topic}] ${event} ` + JSON.stringify(payload)); + logger?.debug(`<< [${topic}] ${event} ` + stringify(payload)); ws.send( - JSON.stringify({ + stringify({ event, ref, topic, @@ -166,9 +165,7 @@ function createServer({ // phx sends this info in each message const { topic, event, payload, ref } = evt; - logger?.debug( - `>> [${topic}] ${event} ${ref} :: ${JSON.stringify(payload)}` - ); + logger?.debug(`>> [${topic}] ${event} ${ref} :: ${stringify(payload)}`); if (events[event]) { // handle system/phoenix events diff --git a/packages/rtm-server/src/util/get-with-reply.ts b/packages/rtm-server/src/util/get-with-reply.ts index cc1dd15f9..ac6b0c778 100644 --- a/packages/rtm-server/src/util/get-with-reply.ts +++ b/packages/rtm-server/src/util/get-with-reply.ts @@ -1,7 +1,7 @@ import { Channel } from '../types'; -export default (channel: Channel, event: string, payload?: any) => - new Promise((resolve) => { +export default (channel: Channel, event: string, payload?: any) => + new Promise((resolve) => { channel.push(event, payload).receive('ok', (evt: any) => { resolve(evt); }); diff --git a/packages/rtm-server/src/util/index.ts b/packages/rtm-server/src/util/index.ts index a9e402621..781e18394 100644 --- a/packages/rtm-server/src/util/index.ts +++ b/packages/rtm-server/src/util/index.ts @@ -1,5 +1,6 @@ import convertAttempt from './convert-attempt'; import tryWithBackoff from './try-with-backoff'; import getWithReply from './get-with-reply'; +import stringify from './stringify'; -export { convertAttempt, tryWithBackoff, getWithReply }; +export { convertAttempt, tryWithBackoff, getWithReply, stringify }; diff --git a/packages/rtm-server/src/util/stringify.ts b/packages/rtm-server/src/util/stringify.ts new file mode 100644 index 000000000..14d514584 --- /dev/null +++ b/packages/rtm-server/src/util/stringify.ts @@ -0,0 +1,9 @@ +import stringify from 'fast-safe-stringify'; + +export default (obj: any): string => + stringify(obj, (_key: string, value: any) => { + if (value instanceof Uint8Array) { + return Array.from(value); + } + return value; + }); diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/rtm-server/test/api/execute.test.ts index 3b902c7b3..591f18027 100644 --- a/packages/rtm-server/test/api/execute.test.ts +++ b/packages/rtm-server/test/api/execute.test.ts @@ -23,6 +23,11 @@ import { } from '../../src/api/execute'; import createMockRTM from '../../src/mock/runtime-manager'; import { mockChannel } from '../../src/mock/sockets'; +import { stringify } from '../../src/util'; + +const enc = new TextEncoder(); + +const toArrayBuffer = (obj: any) => enc.encode(stringify(obj)); test('jobStart should set a run id and active job on state', async (t) => { const plan = { id: 'attempt-1' }; @@ -209,7 +214,7 @@ test('loadState should fetch a dataclip', async (t) => { const channel = mockChannel({ [GET_DATACLIP]: ({ dataclip_id }) => { t.is(dataclip_id, 'xyz'); - return { data: {} }; + return toArrayBuffer({ data: {} }); }, }); @@ -284,7 +289,7 @@ test('execute should lazy-load initial state', async (t) => { [GET_DATACLIP]: (id) => { t.truthy(id); didCallState = true; - return {}; + return toArrayBuffer({}); }, }); const rtm = createMockRTM('rtm'); diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/rtm-server/test/mock/lightning.test.ts index 0ff2629e5..69d104f9e 100644 --- a/packages/rtm-server/test/mock/lightning.test.ts +++ b/packages/rtm-server/test/mock/lightning.test.ts @@ -17,6 +17,8 @@ import { JSONLog } from '@openfn/logger'; const endpoint = 'ws://localhost:7777/api'; +const enc = new TextDecoder('utf-8'); + let server; let client; @@ -234,8 +236,8 @@ test.serial('get attempt data through the attempt channel', async (t) => { server.startAttempt(attempt1.id); const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(GET_ATTEMPT).receive('ok', (p) => { - t.deepEqual(p, attempt1); + channel.push(GET_ATTEMPT).receive('ok', (attempt) => { + t.deepEqual(attempt, attempt1); done(); }); }); @@ -320,7 +322,9 @@ test.serial('get dataclip through the attempt channel', async (t) => { const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(GET_DATACLIP, { id: 'd' }).receive('ok', (result) => { - t.deepEqual(result, dataclips['d']); + const str = enc.decode(new Uint8Array(result)); + const dataclip = JSON.parse(str); + t.deepEqual(dataclip, dataclips['d']); done(); }); }); diff --git a/packages/rtm-server/test/util/stringify.test.ts b/packages/rtm-server/test/util/stringify.test.ts new file mode 100644 index 000000000..81406f2d0 --- /dev/null +++ b/packages/rtm-server/test/util/stringify.test.ts @@ -0,0 +1,24 @@ +import test from 'ava'; + +import stringify from '../../src/util/stringify'; + +test('should stringify an object', (t) => { + const obj = { a: 1 }; + const str = stringify(obj); + + t.is(str, '{"a":1}'); +}); + +test('should stringify a nested object', (t) => { + const obj = { a: { b: 1 } }; + const str = stringify(obj); + + t.is(str, '{"a":{"b":1}}'); +}); + +test('should stringify an ArrayBuffer', (t) => { + const buff = new Uint8Array([42]); + const str = stringify(buff); + + t.is(str, '[42]'); +}); From c412e3ee7ca67b015f7bfa03b6a5ce455e456ace Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 18:04:08 +0100 Subject: [PATCH 098/232] worker: rename rtm-server to ws-worker --- packages/rtm-server/README.md | 59 ------------- packages/rtm-server/notes | 87 ------------------- packages/rtm-server/yargs | 0 packages/runtime-manager/src/rtm.ts | 6 +- packages/ws-worker/README.md | 56 ++++++++++++ .../{rtm-server => ws-worker}/package.json | 6 +- .../{rtm-server => ws-worker}/src/api-rest.ts | 0 .../src/api/connect.ts | 0 .../src/api/execute.ts | 0 .../src/api/start-attempt.ts | 0 .../src/api/workloop.ts | 0 .../{rtm-server => ws-worker}/src/events.ts | 0 .../{rtm-server => ws-worker}/src/index.ts | 0 .../src/middleware/healthcheck.ts | 0 .../src/middleware/workflow.ts | 0 .../src/mock/lightning/api-dev.ts | 0 .../src/mock/lightning/api-sockets.ts | 0 .../src/mock/lightning/index.ts | 0 .../src/mock/lightning/server.ts | 0 .../src/mock/lightning/socket-server.ts | 0 .../src/mock/lightning/start.ts | 0 .../src/mock/lightning/util.ts | 0 .../src/mock/resolvers.ts | 0 .../src/mock/runtime-manager.ts | 0 .../src/mock/sockets.ts | 0 .../{rtm-server => ws-worker}/src/server.ts | 0 .../{rtm-server => ws-worker}/src/start.ts | 0 .../{rtm-server => ws-worker}/src/types.d.ts | 0 .../src/util/convert-attempt.ts | 0 .../src/util/get-with-reply.ts | 0 .../src/util/index.ts | 0 .../src/util/stringify.ts | 0 .../src/util/try-with-backoff.ts | 0 .../src/util/worker-token.ts | 0 .../test/api/connect.test.ts | 0 .../test/api/execute.test.ts | 0 .../test/api/start-attempt.test.ts | 0 .../test/api/workloop.test.ts | 0 .../test/integration.test.ts | 0 .../test/mock/data.ts | 0 .../test/mock/lightning.test.ts | 0 .../test/mock/runtime-manager.test.ts | 0 .../test/mock/socket-server.test.ts | 0 .../test/mock/sockets.test.ts | 0 .../test/server.test.ts | 0 .../test/socket-client.js | 0 .../{rtm-server => ws-worker}/test/util.ts | 0 .../test/util/convert-attempt.test.ts | 0 .../test/util/stringify.test.ts | 0 .../test/util/try-with-backoff.test.ts | 0 .../test/util/worker-token.test.ts | 0 .../{rtm-server => ws-worker}/tsconfig.json | 0 pnpm-lock.yaml | 3 + 53 files changed, 65 insertions(+), 152 deletions(-) delete mode 100644 packages/rtm-server/README.md delete mode 100644 packages/rtm-server/notes delete mode 100644 packages/rtm-server/yargs create mode 100644 packages/ws-worker/README.md rename packages/{rtm-server => ws-worker}/package.json (92%) rename packages/{rtm-server => ws-worker}/src/api-rest.ts (100%) rename packages/{rtm-server => ws-worker}/src/api/connect.ts (100%) rename packages/{rtm-server => ws-worker}/src/api/execute.ts (100%) rename packages/{rtm-server => ws-worker}/src/api/start-attempt.ts (100%) rename packages/{rtm-server => ws-worker}/src/api/workloop.ts (100%) rename packages/{rtm-server => ws-worker}/src/events.ts (100%) rename packages/{rtm-server => ws-worker}/src/index.ts (100%) rename packages/{rtm-server => ws-worker}/src/middleware/healthcheck.ts (100%) rename packages/{rtm-server => ws-worker}/src/middleware/workflow.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/api-dev.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/api-sockets.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/index.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/server.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/socket-server.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/start.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/lightning/util.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/resolvers.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/runtime-manager.ts (100%) rename packages/{rtm-server => ws-worker}/src/mock/sockets.ts (100%) rename packages/{rtm-server => ws-worker}/src/server.ts (100%) rename packages/{rtm-server => ws-worker}/src/start.ts (100%) rename packages/{rtm-server => ws-worker}/src/types.d.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/convert-attempt.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/get-with-reply.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/index.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/stringify.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/try-with-backoff.ts (100%) rename packages/{rtm-server => ws-worker}/src/util/worker-token.ts (100%) rename packages/{rtm-server => ws-worker}/test/api/connect.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/api/execute.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/api/start-attempt.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/api/workloop.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/integration.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/mock/data.ts (100%) rename packages/{rtm-server => ws-worker}/test/mock/lightning.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/mock/runtime-manager.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/mock/socket-server.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/mock/sockets.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/server.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/socket-client.js (100%) rename packages/{rtm-server => ws-worker}/test/util.ts (100%) rename packages/{rtm-server => ws-worker}/test/util/convert-attempt.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/util/stringify.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/util/try-with-backoff.test.ts (100%) rename packages/{rtm-server => ws-worker}/test/util/worker-token.test.ts (100%) rename packages/{rtm-server => ws-worker}/tsconfig.json (100%) diff --git a/packages/rtm-server/README.md b/packages/rtm-server/README.md deleted file mode 100644 index d39649a71..000000000 --- a/packages/rtm-server/README.md +++ /dev/null @@ -1,59 +0,0 @@ -# RTM Server - -The RTM server provides a HTTP interface between Lightning and a runtime manager (RTM). - -This package contains a mock Lightning implementation and a mock runtime manager implementation, allowing lightweight testing of the interfaces between them. - -The RTM server is designed for zero persistence. - -## Architecture - -Lightning will push an Attempt into a queue. - -The RTM server will greedily post Lightning to ask for outstanding attempts, which will returned as JSON objects to the server. - -The Lightning Attempt is converted ta Runtime Execution Plan, and passed to the RTM to execute. - -The server will listen to start, end, and log events in the RTM and POST them back to Lightning. - -## Dev server - -You can start a dev server by running: - -``` -pnpm start:watch -``` - -This will wrap a real runtime manager instance into the server. It will rebuild when the server or RTM code changes (although you'll have to `pnpm build:watch` in `runtime-manager`) - -To connect to a lightning instance, pass the `-l` flag. Use `-l mock` to connect to the default mock server from this repo, or pass your own url. - -The server will create a Runtime Manager instance using the repo at `OPENFN_RTM_REPO_DIR` or `/tmp/openfn/repo`. - -## Lightning Mock - -See `src/mock/lightning/api.ts` for an overview of the expected formal lightning API. This is the API that the RTM server will call. - -Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. - -You can start a Lightning mock server with: -``` -pnpm start:lightning -``` - -This will run on port 8888 [TODO: drop yargs in to customise the port] - -Get the Attempts queue with: -``` -curl http://localhost:8888/api/1/attempts/next -``` -Add an attempt (`{ jobs, triggers, edges }`) to the queue with: -``` -curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" -``` -Get an attempt with -``` -curl http://localhost:8888/api/1/attempts/next/:id -``` - - diff --git a/packages/rtm-server/notes b/packages/rtm-server/notes deleted file mode 100644 index ffcb901eb..000000000 --- a/packages/rtm-server/notes +++ /dev/null @@ -1,87 +0,0 @@ -The Runtime Manager Server provides a HTTP interface between Lightning and a Runtime Manager. - -It is a fairly thin layer between the two systems. - -It should ideally have zero persistence. - -## General Architecture - -Lightning provides an endpoint to fetch any pending jobs from a queue. - -The RTM server polls this endpoint, with a backoff, constantly looking for work. - -Key lifecycle events (start, end, error) are published to Lightning via POST. - -Runtime logs are sent to Lightning in batches via a websocket [TBD] - -## Data Structures - -At the time of writing, the Runtime Manager server uses a Lightning Attempt and a Runtime Execution Plan - -This means it doesn't really "own" any of its datatypes - it just maps from one to another. Which is nice and feels appropriate to the architecture. - -The Attempt is of the form { jobs, triggers, nodes }, a reflection of the underlying workflow model. - -The Runtime must be able to convert a string configuration or state into an object (using a callback provided by the server, which calls out ot lightning to resolve the SHA/id). This happens just-in-time, after a workflow starts but before each job. At the time of writing it cannot do this. The CLI may need to implement similar functionality (it would be super cool for the CLI to be able to call out to lightning to fetch a credential, with zero persistence on the client machine). - -Issues: - -- I dunno, I think it's backwards. Lightning shouldn't expose its own internal data stucture in the attempt. Surely the process of creating an attempt means generating an execution plan from the Workflow model? -- Also, Lightning needs to be able to do this conversion ANYWAY to enable users to download a workflow and run it in the CLI. -- Forunately it's gonna be easy to conver (or not) the structures on the way in. So I'll just take a lighting-style attempt for now and we can drop it out later if we want - -## Zero Persistence - -So the RTM server grabs an attempt, it gets removed form the Lightning queue. - -The runtime starts processing the job. - -Then the host machine dies. Or loses connectivity. Or the runtime crashes, or hangs forever. Something BAD happens that we can't control. - -The RTM server is restarted. Memory is wiped. - -Now what? The jobs in progress are lost forever because nothing was persisted. Lightning is sitting waiting for the server to return a result - which it won't. - -Solutions: - -- Lightning has its own timeout. So if a job takes more than 5 minutes, it'll count as failed. It can be manually retried. This is not unreasonable. -- What if a RTM job returns after 10 minutes? Lightning probably needs to reject it. It actually needs to handshake -- Lightning could maintain a keepalive with the RTM server. When the keepalive dies, Lightning will know which jobs are outstanding. It can re-queue them. - -Other pitfalls: - -- What if the job half finished and posted some information to another server before it died? is re-running harmful? Surely this is up to the job writer to avoid duplicating data. - -## Logging - -The RTM will publish a lot of logs. - -Usually, we can just return all the logs when they're done in batch. - -If someone happens to be watching, we may want to stream logs live. This will happen but less frequently. - -Should lightning and the RTM server maintain a web socket to pipe all logs through? - -Should logs be posted in batches? Every n-ms? - -I'm gonna keep it really simple for now, and send all the logs after completion in a single post. - -For now, we're gonna keep it simple and post all the logs right away. But pass an array for future proofing. - -But there are different types of logs! - -There's system logs and reporting. Stuff that the runtime is doing. - -And there's job logs - things which cem right out of the job vm (including the adaptor) - -The logs from the RTM are different to the CLI logs, but there are similarites. Like the CLI will print out all the versions. How will the RTM do this? What do we actually want to log? And how does it feedback? - -Do we want `log-rtm` and `log-job` ? - -What about compiler logs? The compiler functionality is gonna be a bit different, espeicially when usng the cache. - -Ok, again, I'm gonna keep it simple. Just do log logging. - -# JSON style - -Should the Lightning interfaces use snake case in JSON objects? Probably? They'd be closer to lightning's native style diff --git a/packages/rtm-server/yargs b/packages/rtm-server/yargs deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/runtime-manager/src/rtm.ts b/packages/runtime-manager/src/rtm.ts index b21d0bbcf..bc597f876 100644 --- a/packages/runtime-manager/src/rtm.ts +++ b/packages/runtime-manager/src/rtm.ts @@ -71,13 +71,13 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const events = new EventEmitter(); if (!repoDir) { - if (process.env.OPENFN_RTM_REPO_DIR) { - repoDir = process.env.OPENFN_RTM_REPO_DIR; + if (process.env.OPENFN_RTE_REPO_DIR) { + repoDir = process.env.OPENFN_RTE_REPO_DIR; } else { repoDir = '/tmp/openfn/repo'; logger.warn('Using default repodir'); logger.warn( - 'Set env var OPENFN_RTM_REPO_DIR to use a different directory' + 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' ); } } diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md new file mode 100644 index 000000000..253b11a5c --- /dev/null +++ b/packages/ws-worker/README.md @@ -0,0 +1,56 @@ +# Websocket Worker + +The Websocket Worker `ws-worker` provides a Websocket interface between Lightning and a Runtime Engine. + +It is a fairly thin layer between the two systems, designed to transport messages and convert Lightning data structres into runtime-friendly ones. + +This package contains: + +- A mock Lightning implementation +- A mock runtime engine implementation +- A mock server for phoenix websockets (allowing the phx Socket client to connect and exchange messages) +- A server which connects Lightning to an Engine (exposing dev APIs to http and node.js) + +The mock services allow lightweight and controlled testing of the interfaces between them. + +## Architecture + +Lightning is expected to maintain a queue of attempts. The Worker pulls those attempts from the queue, via websocket, and sends them off to the Engine for execution. + +While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements. + +The RTM server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). + +## Dev server + +You can start a dev server by running: + +``` +pnpm start:watch +``` + +This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `OPENFN_RTE_REPO_DIR` or `/tmp/openfn/repo`. + +To connect to a lightning instance, pass the `-l` flag. Use `-l mock` to connect to the default mock server from this repo, or pass your own url. + +## Lightning Mock + +The key API is in `src/mock/lightning/api-socket/ts`. The `createSocketAPI` function hooks up websockets and binds events to event handlers. It's supposed to be quite declarative so you can track the API quite easily. + +See `src/events.ts` for a typings of the expected event names, payloads and replies. + +Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. + +You can start a Lightning mock server with: + +```s +pnpm start:lightning +``` + +This will run on port 8888 [TODO: drop yargs in to customise the port] + +You can add an attempt (`{ jobs, triggers, edges }`) to the queue with: + +``` +curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" +``` diff --git a/packages/rtm-server/package.json b/packages/ws-worker/package.json similarity index 92% rename from packages/rtm-server/package.json rename to packages/ws-worker/package.json index 5663c4595..a91532b9c 100644 --- a/packages/rtm-server/package.json +++ b/packages/ws-worker/package.json @@ -1,7 +1,7 @@ { - "name": "@openfn/rtm-server", - "version": "0.0.1", - "description": "A REST API wrapper around a runtime manager", + "name": "@openfn/ws-worker", + "version": "0.1.0", + "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "index.js", "type": "module", "private": true, diff --git a/packages/rtm-server/src/api-rest.ts b/packages/ws-worker/src/api-rest.ts similarity index 100% rename from packages/rtm-server/src/api-rest.ts rename to packages/ws-worker/src/api-rest.ts diff --git a/packages/rtm-server/src/api/connect.ts b/packages/ws-worker/src/api/connect.ts similarity index 100% rename from packages/rtm-server/src/api/connect.ts rename to packages/ws-worker/src/api/connect.ts diff --git a/packages/rtm-server/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts similarity index 100% rename from packages/rtm-server/src/api/execute.ts rename to packages/ws-worker/src/api/execute.ts diff --git a/packages/rtm-server/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts similarity index 100% rename from packages/rtm-server/src/api/start-attempt.ts rename to packages/ws-worker/src/api/start-attempt.ts diff --git a/packages/rtm-server/src/api/workloop.ts b/packages/ws-worker/src/api/workloop.ts similarity index 100% rename from packages/rtm-server/src/api/workloop.ts rename to packages/ws-worker/src/api/workloop.ts diff --git a/packages/rtm-server/src/events.ts b/packages/ws-worker/src/events.ts similarity index 100% rename from packages/rtm-server/src/events.ts rename to packages/ws-worker/src/events.ts diff --git a/packages/rtm-server/src/index.ts b/packages/ws-worker/src/index.ts similarity index 100% rename from packages/rtm-server/src/index.ts rename to packages/ws-worker/src/index.ts diff --git a/packages/rtm-server/src/middleware/healthcheck.ts b/packages/ws-worker/src/middleware/healthcheck.ts similarity index 100% rename from packages/rtm-server/src/middleware/healthcheck.ts rename to packages/ws-worker/src/middleware/healthcheck.ts diff --git a/packages/rtm-server/src/middleware/workflow.ts b/packages/ws-worker/src/middleware/workflow.ts similarity index 100% rename from packages/rtm-server/src/middleware/workflow.ts rename to packages/ws-worker/src/middleware/workflow.ts diff --git a/packages/rtm-server/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/api-dev.ts rename to packages/ws-worker/src/mock/lightning/api-dev.ts diff --git a/packages/rtm-server/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/api-sockets.ts rename to packages/ws-worker/src/mock/lightning/api-sockets.ts diff --git a/packages/rtm-server/src/mock/lightning/index.ts b/packages/ws-worker/src/mock/lightning/index.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/index.ts rename to packages/ws-worker/src/mock/lightning/index.ts diff --git a/packages/rtm-server/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/server.ts rename to packages/ws-worker/src/mock/lightning/server.ts diff --git a/packages/rtm-server/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/socket-server.ts rename to packages/ws-worker/src/mock/lightning/socket-server.ts diff --git a/packages/rtm-server/src/mock/lightning/start.ts b/packages/ws-worker/src/mock/lightning/start.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/start.ts rename to packages/ws-worker/src/mock/lightning/start.ts diff --git a/packages/rtm-server/src/mock/lightning/util.ts b/packages/ws-worker/src/mock/lightning/util.ts similarity index 100% rename from packages/rtm-server/src/mock/lightning/util.ts rename to packages/ws-worker/src/mock/lightning/util.ts diff --git a/packages/rtm-server/src/mock/resolvers.ts b/packages/ws-worker/src/mock/resolvers.ts similarity index 100% rename from packages/rtm-server/src/mock/resolvers.ts rename to packages/ws-worker/src/mock/resolvers.ts diff --git a/packages/rtm-server/src/mock/runtime-manager.ts b/packages/ws-worker/src/mock/runtime-manager.ts similarity index 100% rename from packages/rtm-server/src/mock/runtime-manager.ts rename to packages/ws-worker/src/mock/runtime-manager.ts diff --git a/packages/rtm-server/src/mock/sockets.ts b/packages/ws-worker/src/mock/sockets.ts similarity index 100% rename from packages/rtm-server/src/mock/sockets.ts rename to packages/ws-worker/src/mock/sockets.ts diff --git a/packages/rtm-server/src/server.ts b/packages/ws-worker/src/server.ts similarity index 100% rename from packages/rtm-server/src/server.ts rename to packages/ws-worker/src/server.ts diff --git a/packages/rtm-server/src/start.ts b/packages/ws-worker/src/start.ts similarity index 100% rename from packages/rtm-server/src/start.ts rename to packages/ws-worker/src/start.ts diff --git a/packages/rtm-server/src/types.d.ts b/packages/ws-worker/src/types.d.ts similarity index 100% rename from packages/rtm-server/src/types.d.ts rename to packages/ws-worker/src/types.d.ts diff --git a/packages/rtm-server/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts similarity index 100% rename from packages/rtm-server/src/util/convert-attempt.ts rename to packages/ws-worker/src/util/convert-attempt.ts diff --git a/packages/rtm-server/src/util/get-with-reply.ts b/packages/ws-worker/src/util/get-with-reply.ts similarity index 100% rename from packages/rtm-server/src/util/get-with-reply.ts rename to packages/ws-worker/src/util/get-with-reply.ts diff --git a/packages/rtm-server/src/util/index.ts b/packages/ws-worker/src/util/index.ts similarity index 100% rename from packages/rtm-server/src/util/index.ts rename to packages/ws-worker/src/util/index.ts diff --git a/packages/rtm-server/src/util/stringify.ts b/packages/ws-worker/src/util/stringify.ts similarity index 100% rename from packages/rtm-server/src/util/stringify.ts rename to packages/ws-worker/src/util/stringify.ts diff --git a/packages/rtm-server/src/util/try-with-backoff.ts b/packages/ws-worker/src/util/try-with-backoff.ts similarity index 100% rename from packages/rtm-server/src/util/try-with-backoff.ts rename to packages/ws-worker/src/util/try-with-backoff.ts diff --git a/packages/rtm-server/src/util/worker-token.ts b/packages/ws-worker/src/util/worker-token.ts similarity index 100% rename from packages/rtm-server/src/util/worker-token.ts rename to packages/ws-worker/src/util/worker-token.ts diff --git a/packages/rtm-server/test/api/connect.test.ts b/packages/ws-worker/test/api/connect.test.ts similarity index 100% rename from packages/rtm-server/test/api/connect.test.ts rename to packages/ws-worker/test/api/connect.test.ts diff --git a/packages/rtm-server/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts similarity index 100% rename from packages/rtm-server/test/api/execute.test.ts rename to packages/ws-worker/test/api/execute.test.ts diff --git a/packages/rtm-server/test/api/start-attempt.test.ts b/packages/ws-worker/test/api/start-attempt.test.ts similarity index 100% rename from packages/rtm-server/test/api/start-attempt.test.ts rename to packages/ws-worker/test/api/start-attempt.test.ts diff --git a/packages/rtm-server/test/api/workloop.test.ts b/packages/ws-worker/test/api/workloop.test.ts similarity index 100% rename from packages/rtm-server/test/api/workloop.test.ts rename to packages/ws-worker/test/api/workloop.test.ts diff --git a/packages/rtm-server/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts similarity index 100% rename from packages/rtm-server/test/integration.test.ts rename to packages/ws-worker/test/integration.test.ts diff --git a/packages/rtm-server/test/mock/data.ts b/packages/ws-worker/test/mock/data.ts similarity index 100% rename from packages/rtm-server/test/mock/data.ts rename to packages/ws-worker/test/mock/data.ts diff --git a/packages/rtm-server/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts similarity index 100% rename from packages/rtm-server/test/mock/lightning.test.ts rename to packages/ws-worker/test/mock/lightning.test.ts diff --git a/packages/rtm-server/test/mock/runtime-manager.test.ts b/packages/ws-worker/test/mock/runtime-manager.test.ts similarity index 100% rename from packages/rtm-server/test/mock/runtime-manager.test.ts rename to packages/ws-worker/test/mock/runtime-manager.test.ts diff --git a/packages/rtm-server/test/mock/socket-server.test.ts b/packages/ws-worker/test/mock/socket-server.test.ts similarity index 100% rename from packages/rtm-server/test/mock/socket-server.test.ts rename to packages/ws-worker/test/mock/socket-server.test.ts diff --git a/packages/rtm-server/test/mock/sockets.test.ts b/packages/ws-worker/test/mock/sockets.test.ts similarity index 100% rename from packages/rtm-server/test/mock/sockets.test.ts rename to packages/ws-worker/test/mock/sockets.test.ts diff --git a/packages/rtm-server/test/server.test.ts b/packages/ws-worker/test/server.test.ts similarity index 100% rename from packages/rtm-server/test/server.test.ts rename to packages/ws-worker/test/server.test.ts diff --git a/packages/rtm-server/test/socket-client.js b/packages/ws-worker/test/socket-client.js similarity index 100% rename from packages/rtm-server/test/socket-client.js rename to packages/ws-worker/test/socket-client.js diff --git a/packages/rtm-server/test/util.ts b/packages/ws-worker/test/util.ts similarity index 100% rename from packages/rtm-server/test/util.ts rename to packages/ws-worker/test/util.ts diff --git a/packages/rtm-server/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts similarity index 100% rename from packages/rtm-server/test/util/convert-attempt.test.ts rename to packages/ws-worker/test/util/convert-attempt.test.ts diff --git a/packages/rtm-server/test/util/stringify.test.ts b/packages/ws-worker/test/util/stringify.test.ts similarity index 100% rename from packages/rtm-server/test/util/stringify.test.ts rename to packages/ws-worker/test/util/stringify.test.ts diff --git a/packages/rtm-server/test/util/try-with-backoff.test.ts b/packages/ws-worker/test/util/try-with-backoff.test.ts similarity index 100% rename from packages/rtm-server/test/util/try-with-backoff.test.ts rename to packages/ws-worker/test/util/try-with-backoff.test.ts diff --git a/packages/rtm-server/test/util/worker-token.test.ts b/packages/ws-worker/test/util/worker-token.test.ts similarity index 100% rename from packages/rtm-server/test/util/worker-token.test.ts rename to packages/ws-worker/test/util/worker-token.test.ts diff --git a/packages/rtm-server/tsconfig.json b/packages/ws-worker/tsconfig.json similarity index 100% rename from packages/rtm-server/tsconfig.json rename to packages/ws-worker/tsconfig.json diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 78b73178a..b5a56274d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -387,6 +387,9 @@ importers: '@types/ws': specifier: ^8.5.6 version: 8.5.6 + fast-safe-stringify: + specifier: ^2.1.1 + version: 2.1.1 jose: specifier: ^4.14.6 version: 4.14.6 From 81cae05664c4bcccad38db6e3a7522fdeb97857e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 18:26:57 +0100 Subject: [PATCH 099/232] worker: rename internal rtm -> engine or rte --- packages/ws-worker/README.md | 2 +- packages/ws-worker/src/api/execute.ts | 10 +-- .../ws-worker/src/mock/lightning/server.ts | 4 +- .../ws-worker/src/mock/lightning/start.ts | 2 +- packages/ws-worker/src/mock/resolvers.ts | 2 +- .../{runtime-manager.ts => runtime-engine.ts} | 14 ++-- packages/ws-worker/src/server.ts | 15 ++-- packages/ws-worker/src/start.ts | 20 ++--- packages/ws-worker/test/api/execute.test.ts | 22 ++--- packages/ws-worker/test/integration.test.ts | 14 ++-- ...time-manager.test.ts => runtime-engine.ts} | 80 +++++++++---------- packages/ws-worker/test/server.test.ts | 12 +-- packages/ws-worker/test/util.ts | 4 +- 13 files changed, 100 insertions(+), 101 deletions(-) rename packages/ws-worker/src/mock/{runtime-manager.ts => runtime-engine.ts} (93%) rename packages/ws-worker/test/mock/{runtime-manager.test.ts => runtime-engine.ts} (68%) diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index 253b11a5c..84df169dd 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -19,7 +19,7 @@ Lightning is expected to maintain a queue of attempts. The Worker pulls those at While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements. -The RTM server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). +The ws-worker server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). ## Dev server diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index f42f03c8b..fe3bf24c9 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -1,6 +1,6 @@ // TODO not crazy about this file name // This is the module responsible for interfacing between the Lightning websocket -// and the RTM +// and the rutnime engine // It's the actual meat and potatoes of the implementation // You can almost read this as a binding function and a bunch of handlers // it isn't an actual worker, but a BRIDGE between a worker and lightning @@ -43,7 +43,7 @@ export type AttemptState = { // pass a web socket connected to the attempt channel // this thing will do all the work -export function execute(channel: Channel, rtm, plan: ExecutionPlan) { +export function execute(channel: Channel, engine, plan: ExecutionPlan) { return new Promise((resolve) => { // TODO add proper logger (maybe channel, rtm and logger comprise a context object) // tracking state for this attempt @@ -54,7 +54,7 @@ export function execute(channel: Channel, rtm, plan: ExecutionPlan) { // TODO // const context = { channel, state, logger } - rtm.listen(plan.id, { + engine.listen(plan.id, { // TODO load event types from runtime-manager 'workflow-start': (evt: any) => onWorkflowStart(channel), 'job-start': (evt: any) => onJobStart(channel, state, evt), @@ -71,7 +71,7 @@ export function execute(channel: Channel, rtm, plan: ExecutionPlan) { credential: (id: string) => loadCredential(channel, id), }; - rtm.execute(plan, resolvers); + engine.execute(plan, resolvers); }); } @@ -96,7 +96,7 @@ export function onJobStart( export function onJobComplete( channel: Channel, state: AttemptState, - evt: any // TODO need to type the RTM events nicely + evt: any // TODO need to type the engine events nicely ) { channel.push(RUN_COMPLETE, { run_id: state.activeRun!, diff --git a/packages/ws-worker/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts index ad536f696..a4986b1d2 100644 --- a/packages/ws-worker/src/mock/lightning/server.ts +++ b/packages/ws-worker/src/mock/lightning/server.ts @@ -32,11 +32,11 @@ export type ServerState = { dataclips: Record; // Tracking state of known attempts - // TODO include the rtm id and token + // TODO include the engine id and token pending: Record; // Track all completed attempts here - results: Record; + results: Record; // event emitter for debugging and observability events: EventEmitter; diff --git a/packages/ws-worker/src/mock/lightning/start.ts b/packages/ws-worker/src/mock/lightning/start.ts index 6cb424053..557d7b07f 100644 --- a/packages/ws-worker/src/mock/lightning/start.ts +++ b/packages/ws-worker/src/mock/lightning/start.ts @@ -10,7 +10,7 @@ type Args = { }; const args = yargs(hideBin(process.argv)) - .command('server', 'Start a runtime manager server') + .command('server', 'Start a mock lighting server') .option('port', { alias: 'p', description: 'Port to run the server on', diff --git a/packages/ws-worker/src/mock/resolvers.ts b/packages/ws-worker/src/mock/resolvers.ts index f328e0876..871c88f14 100644 --- a/packages/ws-worker/src/mock/resolvers.ts +++ b/packages/ws-worker/src/mock/resolvers.ts @@ -1,5 +1,5 @@ import type { State, Credential } from '../types'; -import { LazyResolvers } from './runtime-manager'; +import { LazyResolvers } from './runtime-engine'; const mockResolveCredential = (_credId: string) => new Promise((resolve) => diff --git a/packages/ws-worker/src/mock/runtime-manager.ts b/packages/ws-worker/src/mock/runtime-engine.ts similarity index 93% rename from packages/ws-worker/src/mock/runtime-manager.ts rename to packages/ws-worker/src/mock/runtime-engine.ts index 841630271..3027f331d 100644 --- a/packages/ws-worker/src/mock/runtime-manager.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -4,10 +4,11 @@ import type { ExecutionPlan } from '@openfn/runtime'; import type { State, Credential } from '../types'; import mockResolvers from './resolvers'; -// A mock runtime manager +// A mock runtime engine + // Runs ExecutionPlans(XPlans) in worker threads // May need to lazy-load resources -// The mock RTM will return expression JSON as state +// The mock engine will return expression JSON as state type Resolver = (id: string) => Promise; @@ -19,7 +20,7 @@ export type LazyResolvers = { expressions?: Resolver; }; -export type RTMEvent = +export type EngineEvent = | 'job-start' | 'job-complete' | 'log' // this is a log from inside the VM @@ -67,7 +68,7 @@ function createMock(serverId?: string) { const bus = new EventEmitter(); const listeners: Record = {}; - const dispatch = (type: RTMEvent, args?: any) => { + const dispatch = (type: EngineEvent, args?: any) => { // console.log(' > ', type, args); if (args.workflowId) { listeners[args.workflowId]?.[type]?.(args); @@ -76,9 +77,10 @@ function createMock(serverId?: string) { bus.emit(type, args); }; - const on = (event: RTMEvent, fn: (evt: any) => void) => bus.on(event, fn); + const on = (event: EngineEvent, fn: (evt: any) => void) => bus.on(event, fn); - const once = (event: RTMEvent, fn: (evt: any) => void) => bus.once(event, fn); + const once = (event: EngineEvent, fn: (evt: any) => void) => + bus.once(event, fn); // Listens to events for a particular workflow/execution plan // TODO: Listeners will be removed when the plan is complete (?) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index cc9dcf967..2b0891c61 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -15,13 +15,12 @@ type ServerOptions = { maxWorkflows?: number; port?: number; lightning?: string; // url to lightning instance - rtm?: any; logger?: Logger; - secret: string; // worker secret + secret?: string; // worker secret }; -function createServer(rtm: any, options: ServerOptions = {}) { +function createServer(engine: any, options: ServerOptions = {}) { const logger = options.logger || createMockLogger(); const port = options.port || 1234; @@ -39,7 +38,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { createRestAPI(app, logger); app.listen(port); - logger.success('RTM server listening on', port); + logger.success('ws-worker listening on', port); (app as any).destroy = () => { // TODO close the work loop @@ -53,7 +52,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { if (options.lightning) { logger.log('Starting work loop at', options.lightning); - connectToLightning(options.lightning, rtm.id, options.secret).then( + connectToLightning(options.lightning, engine.id, options.secret!).then( ({ socket, channel }) => { const startAttempt = async ({ id, token }: StartAttemptArgs) => { const { channel: attemptChannel, plan } = await joinAttemptChannel( @@ -61,7 +60,7 @@ function createServer(rtm: any, options: ServerOptions = {}) { token, id ); - execute(attemptChannel, rtm, plan); + execute(attemptChannel, engine, plan); }; // TODO maybe pull this logic out so we can test it? @@ -77,8 +76,8 @@ function createServer(rtm: any, options: ServerOptions = {}) { } // TMP doing this for tests but maybe its better done externally - app.on = (...args) => rtm.on(...args); - app.once = (...args) => rtm.once(...args); + app.on = (...args) => engine.on(...args); + app.once = (...args) => engine.once(...args); return app; } diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 8ad398aee..eb15955c5 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -3,9 +3,9 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import createLogger from '@openfn/logger'; -import createRTM from '@openfn/runtime-manager'; -import createMockRTM from './mock/runtime-manager'; -import createRTMServer from './server'; +// import createRTM from '@openfn/runtime-engine'; +import createMockRTE from './mock/runtime-engine'; +import createWorker from './server'; type Args = { _: string[]; @@ -18,7 +18,7 @@ type Args = { const logger = createLogger('SRV', { level: 'info' }); const args = yargs(hideBin(process.argv)) - .command('server', 'Start a runtime manager server') + .command('server', 'Start a ws-worker server') .option('port', { alias: 'p', description: 'Port to run the server on', @@ -50,20 +50,20 @@ if (args.lightning === 'mock') { args.secret = process.env.WORKER_SECRET; } -// TODO the rtm needs to take callbacks to load credential, and load state +// TODO the engine needs to take callbacks to load credential, and load state // these in turn should utilise the websocket // So either the server creates the runtime (which seems reasonable acutally?) -// Or the server calls a setCalbacks({ credential, state }) function on the RTM +// Or the server calls a setCalbacks({ credential, state }) function on the engine // Each of these takes the attemptId as the firsdt argument // credential and state will lookup the right channel -// const rtm = createRTM('rtm', { repoDir: args.repoDir }); -// logger.debug('RTM created'); +// const engine = createEngine('rte', { repoDir: args.repoDir }); +// logger.debug('engine created'); // use the mock rtm for now -const rtm = createMockRTM('rtm'); +const engine = createMockRTE('rtm'); logger.debug('Mock RTM created'); -createRTMServer(rtm, { +createWorker(engine, { port: args.port, lightning: args.lightning, logger, diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 591f18027..2cbff53c4 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -21,7 +21,7 @@ import { loadState, loadCredential, } from '../../src/api/execute'; -import createMockRTM from '../../src/mock/runtime-manager'; +import createMockRTE from '../../src/mock/runtime-engine'; import { mockChannel } from '../../src/mock/sockets'; import { stringify } from '../../src/util'; @@ -237,7 +237,7 @@ test('loadCredential should fetch a credential', async (t) => { test('execute should return the final result', async (t) => { const channel = mockChannel(); - const rtm = createMockRTM(); + const engine = createMockRTE(); const plan = { id: 'a', @@ -248,12 +248,12 @@ test('execute should return the final result', async (t) => { ], }; - const result = await execute(channel, rtm, plan); + const result = await execute(channel, engine, plan); t.deepEqual(result, { done: true }); }); -// TODO this is more of an RTM test really, but worth having I suppose +// TODO this is more of an engine test really, but worth having I suppose test('execute should lazy-load a credential', async (t) => { let didCallCredentials = false; @@ -264,7 +264,7 @@ test('execute should lazy-load a credential', async (t) => { return {}; }, }); - const rtm = createMockRTM('rtm'); + const engine = createMockRTE('rte'); const plan = { id: 'a', @@ -276,12 +276,12 @@ test('execute should lazy-load a credential', async (t) => { ], }; - await execute(channel, rtm, plan); + await execute(channel, engine, plan); t.true(didCallCredentials); }); -// TODO this is more of an RTM test really, but worth having I suppose +// TODO this is more of an engine test really, but worth having I suppose test('execute should lazy-load initial state', async (t) => { let didCallState = false; @@ -292,7 +292,7 @@ test('execute should lazy-load initial state', async (t) => { return toArrayBuffer({}); }, }); - const rtm = createMockRTM('rtm'); + const engine = createMockRTE('rte'); const plan = { id: 'a', @@ -304,7 +304,7 @@ test('execute should lazy-load initial state', async (t) => { ], }; - await execute(channel, rtm, plan); + await execute(channel, engine, plan); t.true(didCallState); }); @@ -312,7 +312,7 @@ test('execute should lazy-load initial state', async (t) => { test('execute should call all events on the socket', async (t) => { const events = {}; - const rtm = createMockRTM(); + const engine = createMockRTE(); const toEventMap = (obj, evt: string) => { obj[evt] = (e) => { @@ -346,7 +346,7 @@ test('execute should call all events on the socket', async (t) => { ], }; - await execute(channel, rtm, plan); + await execute(channel, engine, plan); // check result is what we expect diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 267863eea..6c297a994 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -1,25 +1,23 @@ /* - * Tests of Lightning-RTM server integration, from Lightning's perspective + * Tests of Lightning-Engine server integration, from Lightning's perspective */ import test from 'ava'; -import createRTMServer from '../src/server'; +import createWorkerServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; -import createMockRTM from '../src/mock/runtime-manager'; - -import { wait, waitForEvent } from './util'; +import createMockRTE from '../src/mock/runtime-engine'; let lng; -let rtm; +let engine; const urls = { - rtm: 'http://localhost:4567', + engine: 'http://localhost:4567', lng: 'http://localhost:7654', }; test.before(() => { lng = createLightningServer({ port: 7654 }); - rtm = createRTMServer(createMockRTM('rtm'), { + engine = createWorkerServer(createMockRTE('engine'), { port: 4567, lightning: urls.lng, }); diff --git a/packages/ws-worker/test/mock/runtime-manager.test.ts b/packages/ws-worker/test/mock/runtime-engine.ts similarity index 68% rename from packages/ws-worker/test/mock/runtime-manager.test.ts rename to packages/ws-worker/test/mock/runtime-engine.ts index 4fc0052df..6ba8a9d78 100644 --- a/packages/ws-worker/test/mock/runtime-manager.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.ts @@ -4,7 +4,7 @@ import create, { JobStartEvent, WorkflowCompleteEvent, WorkflowStartEvent, -} from '../../src/mock/runtime-manager'; +} from '../../src/mock/runtime-engine'; import type { ExecutionPlan } from '@openfn/runtime'; import { waitForEvent, clone } from '../util'; // ??? @@ -19,10 +19,10 @@ const sampleWorkflow = { ], } as ExecutionPlan; -test('mock runtime manager should have an id', (t) => { - const rtm = create('22'); - const keys = Object.keys(rtm); - t.assert(rtm.id == '22'); +test('mock runtime engine should have an id', (t) => { + const engine = create('22'); + const keys = Object.keys(engine); + t.assert(engine.id == '22'); // No need to test the full API, just make sure it smells right t.assert(keys.includes('on')); @@ -30,36 +30,36 @@ test('mock runtime manager should have an id', (t) => { }); test('getStatus() should should have no active workflows', (t) => { - const rtm = create(); - const { active } = rtm.getStatus(); + const engine = create(); + const { active } = engine.getStatus(); t.is(active, 0); }); test('Dispatch start events for a new workflow', async (t) => { - const rtm = create(); + const engine = create(); - rtm.execute(sampleWorkflow); - const evt = await waitForEvent(rtm, 'workflow-start'); + engine.execute(sampleWorkflow); + const evt = await waitForEvent(engine, 'workflow-start'); t.truthy(evt); t.is(evt.workflowId, 'w1'); }); test('getStatus should report one active workflow', async (t) => { - const rtm = create(); - rtm.execute(sampleWorkflow); + const engine = create(); + engine.execute(sampleWorkflow); - const { active } = rtm.getStatus(); + const { active } = engine.getStatus(); t.is(active, 1); }); test('Dispatch complete events when a workflow completes', async (t) => { - const rtm = create(); + const engine = create(); - rtm.execute(sampleWorkflow); + engine.execute(sampleWorkflow); const evt = await waitForEvent( - rtm, + engine, 'workflow-complete' ); @@ -69,20 +69,20 @@ test('Dispatch complete events when a workflow completes', async (t) => { }); test('Dispatch start events for a job', async (t) => { - const rtm = create(); + const engine = create(); - rtm.execute(sampleWorkflow); - const evt = await waitForEvent(rtm, 'job-start'); + engine.execute(sampleWorkflow); + const evt = await waitForEvent(engine, 'job-start'); t.truthy(evt); t.is(evt.workflowId, 'w1'); t.is(evt.jobId, 'j1'); }); test('Dispatch complete events for a job', async (t) => { - const rtm = create(); + const engine = create(); - rtm.execute(sampleWorkflow); - const evt = await waitForEvent(rtm, 'job-complete'); + engine.execute(sampleWorkflow); + const evt = await waitForEvent(engine, 'job-complete'); t.truthy(evt); t.is(evt.workflowId, 'w1'); t.is(evt.jobId, 'j1'); @@ -90,26 +90,26 @@ test('Dispatch complete events for a job', async (t) => { }); test('mock should evaluate expressions as JSON', async (t) => { - const rtm = create(); + const engine = create(); - rtm.execute(sampleWorkflow); + engine.execute(sampleWorkflow); const evt = await waitForEvent( - rtm, + engine, 'workflow-complete' ); t.deepEqual(evt.state, { x: 10 }); }); test('mock should dispatch log events when evaluating JSON', async (t) => { - const rtm = create(); + const engine = create(); const logs = []; - rtm.on('log', (l) => { + engine.on('log', (l) => { logs.push(l); }); - rtm.execute(sampleWorkflow); - await waitForEvent(rtm, 'workflow-complete'); + engine.execute(sampleWorkflow); + await waitForEvent(engine, 'workflow-complete'); t.deepEqual(logs[0].message, ['Running job j1']); t.deepEqual(logs[1].message, ['Parsing expression as JSON state']); @@ -125,16 +125,16 @@ test('resolve credential before job-start if credential is a string', async (t) return {}; }; - const rtm = create('1'); + const engine = create('1'); // @ts-ignore - rtm.execute(wf, { credential }); + engine.execute(wf, { credential }); - await waitForEvent(rtm, 'job-start'); + await waitForEvent(engine, 'job-start'); t.true(didCallCredentials); }); test('listen to events', async (t) => { - const rtm = create(); + const engine = create(); const called = { 'job-start': false, @@ -144,7 +144,7 @@ test('listen to events', async (t) => { 'workflow-complete': false, }; - rtm.listen(sampleWorkflow.id, { + engine.listen(sampleWorkflow.id, { 'job-start': ({ workflowId, jobId }) => { called['job-start'] = true; t.is(workflowId, sampleWorkflow.id); @@ -171,21 +171,21 @@ test('listen to events', async (t) => { }, }); - rtm.execute(sampleWorkflow); - await waitForEvent(rtm, 'workflow-complete'); + engine.execute(sampleWorkflow); + await waitForEvent(engine, 'workflow-complete'); t.assert(Object.values(called).every((v) => v === true)); }); test('only listen to events for the correct workflow', async (t) => { - const rtm = create(); + const engine = create(); - rtm.listen('bobby mcgee', { + engine.listen('bobby mcgee', { 'workflow-start': ({ workflowId }) => { throw new Error('should not have called this!!'); }, }); - rtm.execute(sampleWorkflow); - await waitForEvent(rtm, 'workflow-complete'); + engine.execute(sampleWorkflow); + await waitForEvent(engine, 'workflow-complete'); t.pass(); }); diff --git a/packages/ws-worker/test/server.test.ts b/packages/ws-worker/test/server.test.ts index 5917aeb92..aa67b5115 100644 --- a/packages/ws-worker/test/server.test.ts +++ b/packages/ws-worker/test/server.test.ts @@ -3,22 +3,22 @@ import WebSocket, { WebSocketServer } from 'ws'; import createServer from '../src/server'; import connectToLightning from '../src/api/connect'; -import createMockRTM from '../src/mock/runtime-manager'; +import createMockRTE from '../src/mock/runtime-engine'; import { sleep } from './util'; import { mockChannel, mockSocket } from '../src/mock/sockets'; import { CLAIM } from '../src/events'; -// Unit tests against the RTM web server +// Unit tests against the worker server // I don't think there will ever be much here because the server is mostly a pull -let rtm; +let engine; let server; let cancel; const url = 'http://localhost:7777'; test.beforeEach(() => { - rtm = createMockRTM(); + engine = createMockRTE(); }); test.afterEach(() => { @@ -27,7 +27,7 @@ test.afterEach(() => { }); test.skip('healthcheck', async (t) => { - const server = createServer(rtm, { port: 7777 }); + const server = createServer(engine, { port: 7777 }); const result = await fetch(`${url}/healthcheck`); t.is(result.status, 200); const body = await result.text(); @@ -53,7 +53,7 @@ test.skip('healthcheck', async (t) => { // didSayHello = true; // }); -// rtm = createMockRTM(); +// rtm = createMockRTE(); // server = createServer(rtm, { // port: 7777, // lightning: 'ws://localhost:8080', diff --git a/packages/ws-worker/test/util.ts b/packages/ws-worker/test/util.ts index 33d822a42..2ec1388f7 100644 --- a/packages/ws-worker/test/util.ts +++ b/packages/ws-worker/test/util.ts @@ -18,9 +18,9 @@ export const wait = (fn, maxAttempts = 100) => export const clone = (obj) => JSON.parse(JSON.stringify(obj)); -export const waitForEvent = (rtm, eventName) => +export const waitForEvent = (engine, eventName) => new Promise((resolve) => { - rtm.once(eventName, (e) => { + engine.once(eventName, (e) => { resolve(e); }); }); From b4ece5ea9f9a0de3458db5c2b9c3eb4eaa482891 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 28 Sep 2023 18:35:59 +0100 Subject: [PATCH 100/232] engine: quick refactor from rtm to engine-multi --- .../CHANGELOG.md | 0 packages/engine-multi/README.md | 7 + .../package.json | 4 +- .../src/rtm.ts => engine-multi/src/engine.ts} | 0 .../src/events.ts | 0 packages/engine-multi/src/index.ts | 3 + .../src/jobs/slow-random.js | 0 .../src/mock-worker.ts | 0 .../src/runners/autoinstall.ts | 0 .../src/runners/compile.ts | 0 .../src/runners/execute.ts | 0 .../src/worker-helper.ts | 0 .../src/worker.ts | 0 .../test/engine.ts} | 0 .../test/jobs/slow-random.test.ts | 0 .../test/mock-worker.test.ts | 0 .../test/runners/autoinstall.test.ts | 0 .../test/util.ts | 0 .../test/worker-pool.test.ts | 0 .../tsconfig.json | 0 packages/runtime-manager/README.md | 31 --- packages/runtime-manager/src/index.ts | 3 - packages/ws-worker/package.json | 2 +- pnpm-lock.yaml | 225 +++++++++--------- 24 files changed, 125 insertions(+), 150 deletions(-) rename packages/{runtime-manager => engine-multi}/CHANGELOG.md (100%) create mode 100644 packages/engine-multi/README.md rename packages/{runtime-manager => engine-multi}/package.json (91%) rename packages/{runtime-manager/src/rtm.ts => engine-multi/src/engine.ts} (100%) rename packages/{runtime-manager => engine-multi}/src/events.ts (100%) create mode 100644 packages/engine-multi/src/index.ts rename packages/{runtime-manager => engine-multi}/src/jobs/slow-random.js (100%) rename packages/{runtime-manager => engine-multi}/src/mock-worker.ts (100%) rename packages/{runtime-manager => engine-multi}/src/runners/autoinstall.ts (100%) rename packages/{runtime-manager => engine-multi}/src/runners/compile.ts (100%) rename packages/{runtime-manager => engine-multi}/src/runners/execute.ts (100%) rename packages/{runtime-manager => engine-multi}/src/worker-helper.ts (100%) rename packages/{runtime-manager => engine-multi}/src/worker.ts (100%) rename packages/{runtime-manager/test/rtm.test.ts => engine-multi/test/engine.ts} (100%) rename packages/{runtime-manager => engine-multi}/test/jobs/slow-random.test.ts (100%) rename packages/{runtime-manager => engine-multi}/test/mock-worker.test.ts (100%) rename packages/{runtime-manager => engine-multi}/test/runners/autoinstall.test.ts (100%) rename packages/{runtime-manager => engine-multi}/test/util.ts (100%) rename packages/{runtime-manager => engine-multi}/test/worker-pool.test.ts (100%) rename packages/{runtime-manager => engine-multi}/tsconfig.json (100%) delete mode 100644 packages/runtime-manager/README.md delete mode 100644 packages/runtime-manager/src/index.ts diff --git a/packages/runtime-manager/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md similarity index 100% rename from packages/runtime-manager/CHANGELOG.md rename to packages/engine-multi/CHANGELOG.md diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md new file mode 100644 index 000000000..cd4a25942 --- /dev/null +++ b/packages/engine-multi/README.md @@ -0,0 +1,7 @@ +## Multi-process engine + +A runtime engine which runs multiple jobs in worker threads. + +A long-running node service, suitable for integration with a Worker, for executing workflows. + +Docs to follow diff --git a/packages/runtime-manager/package.json b/packages/engine-multi/package.json similarity index 91% rename from packages/runtime-manager/package.json rename to packages/engine-multi/package.json index da62d8d40..70730caca 100644 --- a/packages/runtime-manager/package.json +++ b/packages/engine-multi/package.json @@ -1,7 +1,7 @@ { - "name": "@openfn/runtime-manager", + "name": "@openfn/engine-multi", "version": "0.0.41", - "description": "An example runtime manager service.", + "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", "private": true, diff --git a/packages/runtime-manager/src/rtm.ts b/packages/engine-multi/src/engine.ts similarity index 100% rename from packages/runtime-manager/src/rtm.ts rename to packages/engine-multi/src/engine.ts diff --git a/packages/runtime-manager/src/events.ts b/packages/engine-multi/src/events.ts similarity index 100% rename from packages/runtime-manager/src/events.ts rename to packages/engine-multi/src/events.ts diff --git a/packages/engine-multi/src/index.ts b/packages/engine-multi/src/index.ts new file mode 100644 index 000000000..e4f27252e --- /dev/null +++ b/packages/engine-multi/src/index.ts @@ -0,0 +1,3 @@ +import createEngine from './engine'; + +export default createEngine; diff --git a/packages/runtime-manager/src/jobs/slow-random.js b/packages/engine-multi/src/jobs/slow-random.js similarity index 100% rename from packages/runtime-manager/src/jobs/slow-random.js rename to packages/engine-multi/src/jobs/slow-random.js diff --git a/packages/runtime-manager/src/mock-worker.ts b/packages/engine-multi/src/mock-worker.ts similarity index 100% rename from packages/runtime-manager/src/mock-worker.ts rename to packages/engine-multi/src/mock-worker.ts diff --git a/packages/runtime-manager/src/runners/autoinstall.ts b/packages/engine-multi/src/runners/autoinstall.ts similarity index 100% rename from packages/runtime-manager/src/runners/autoinstall.ts rename to packages/engine-multi/src/runners/autoinstall.ts diff --git a/packages/runtime-manager/src/runners/compile.ts b/packages/engine-multi/src/runners/compile.ts similarity index 100% rename from packages/runtime-manager/src/runners/compile.ts rename to packages/engine-multi/src/runners/compile.ts diff --git a/packages/runtime-manager/src/runners/execute.ts b/packages/engine-multi/src/runners/execute.ts similarity index 100% rename from packages/runtime-manager/src/runners/execute.ts rename to packages/engine-multi/src/runners/execute.ts diff --git a/packages/runtime-manager/src/worker-helper.ts b/packages/engine-multi/src/worker-helper.ts similarity index 100% rename from packages/runtime-manager/src/worker-helper.ts rename to packages/engine-multi/src/worker-helper.ts diff --git a/packages/runtime-manager/src/worker.ts b/packages/engine-multi/src/worker.ts similarity index 100% rename from packages/runtime-manager/src/worker.ts rename to packages/engine-multi/src/worker.ts diff --git a/packages/runtime-manager/test/rtm.test.ts b/packages/engine-multi/test/engine.ts similarity index 100% rename from packages/runtime-manager/test/rtm.test.ts rename to packages/engine-multi/test/engine.ts diff --git a/packages/runtime-manager/test/jobs/slow-random.test.ts b/packages/engine-multi/test/jobs/slow-random.test.ts similarity index 100% rename from packages/runtime-manager/test/jobs/slow-random.test.ts rename to packages/engine-multi/test/jobs/slow-random.test.ts diff --git a/packages/runtime-manager/test/mock-worker.test.ts b/packages/engine-multi/test/mock-worker.test.ts similarity index 100% rename from packages/runtime-manager/test/mock-worker.test.ts rename to packages/engine-multi/test/mock-worker.test.ts diff --git a/packages/runtime-manager/test/runners/autoinstall.test.ts b/packages/engine-multi/test/runners/autoinstall.test.ts similarity index 100% rename from packages/runtime-manager/test/runners/autoinstall.test.ts rename to packages/engine-multi/test/runners/autoinstall.test.ts diff --git a/packages/runtime-manager/test/util.ts b/packages/engine-multi/test/util.ts similarity index 100% rename from packages/runtime-manager/test/util.ts rename to packages/engine-multi/test/util.ts diff --git a/packages/runtime-manager/test/worker-pool.test.ts b/packages/engine-multi/test/worker-pool.test.ts similarity index 100% rename from packages/runtime-manager/test/worker-pool.test.ts rename to packages/engine-multi/test/worker-pool.test.ts diff --git a/packages/runtime-manager/tsconfig.json b/packages/engine-multi/tsconfig.json similarity index 100% rename from packages/runtime-manager/tsconfig.json rename to packages/engine-multi/tsconfig.json diff --git a/packages/runtime-manager/README.md b/packages/runtime-manager/README.md deleted file mode 100644 index 7335ff158..000000000 --- a/packages/runtime-manager/README.md +++ /dev/null @@ -1,31 +0,0 @@ -## Runtime Manager - -An example runtime manager service. - -The runtime manager is designed as a long running node service that runs jobs as worker threads. - -## Usage - -To integrate the manager into your own application: - -1. Create a manager: - -```js -const m = Manager(); -``` - -2. Register jobs (as DSL strings which will be compiled) - -```js -m.registerJob('my_job', 'get(state.url)'); -``` - -3. Run the job - -```js -const report = await m.run('my_job'); -``` - -The report object reports the status, duration, startTime and result of the job. - -The job will soon expose an event emitter so that you can subscribe to individual events. diff --git a/packages/runtime-manager/src/index.ts b/packages/runtime-manager/src/index.ts deleted file mode 100644 index db8e8169e..000000000 --- a/packages/runtime-manager/src/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import createRTM from './rtm'; - -export default createRTM; diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index a91532b9c..a8eba7bac 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -20,7 +20,7 @@ "@koa/router": "^12.0.0", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", - "@openfn/runtime-manager": "workspace:*", + "@openfn/engine-multi": "workspace:*", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5a56274d..747bf6e87 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -333,6 +333,61 @@ importers: specifier: ^7.2.0 version: 7.2.0(ts-node@10.9.1)(typescript@5.1.6) + packages/engine-multi: + dependencies: + '@openfn/compiler': + specifier: workspace:* + version: link:../compiler + '@openfn/language-common': + specifier: 2.0.0-rc3 + version: 2.0.0-rc3 + '@openfn/logger': + specifier: workspace:* + version: link:../logger + '@openfn/runtime': + specifier: workspace:* + version: link:../runtime + koa: + specifier: ^2.13.4 + version: 2.13.4 + workerpool: + specifier: ^6.2.1 + version: 6.2.1 + devDependencies: + '@types/koa': + specifier: ^2.13.5 + version: 2.13.5 + '@types/node': + specifier: ^18.15.13 + version: 18.15.13 + '@types/nodemon': + specifier: ^1.19.2 + version: 1.19.2 + '@types/workerpool': + specifier: ^6.1.0 + version: 6.1.0 + ava: + specifier: 5.3.1 + version: 5.3.1 + nodemon: + specifier: ^2.0.19 + version: 2.0.19 + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@18.15.13)(typescript@5.1.6) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + tsm: + specifier: ^2.2.2 + version: 2.2.2 + tsup: + specifier: ^7.2.0 + version: 7.2.0(ts-node@10.9.1)(typescript@5.1.6) + typescript: + specifier: ^5.1.6 + version: 5.1.6 + packages/logger: dependencies: '@inquirer/confirm': @@ -367,20 +422,63 @@ importers: specifier: ^5.1.6 version: 5.1.6 - packages/rtm-server: + packages/runtime: + dependencies: + '@openfn/logger': + specifier: workspace:* + version: link:../logger + fast-safe-stringify: + specifier: ^2.1.1 + version: 2.1.1 + semver: + specifier: ^7.5.4 + version: 7.5.4 + devDependencies: + '@openfn/language-common': + specifier: 2.0.0-rc3 + version: 2.0.0-rc3 + '@types/mock-fs': + specifier: ^4.13.1 + version: 4.13.1 + '@types/node': + specifier: ^18.15.13 + version: 18.15.13 + '@types/semver': + specifier: ^7.5.0 + version: 7.5.0 + ava: + specifier: 5.3.1 + version: 5.3.1 + mock-fs: + specifier: ^5.1.4 + version: 5.1.4 + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@18.15.13)(typescript@5.1.6) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + tsup: + specifier: ^7.2.0 + version: 7.2.0(ts-node@10.9.1)(typescript@5.1.6) + typescript: + specifier: ^5.1.6 + version: 5.1.6 + + packages/ws-worker: dependencies: '@koa/router': specifier: ^12.0.0 version: 12.0.0 + '@openfn/engine-multi': + specifier: workspace:* + version: link:../engine-multi '@openfn/logger': specifier: workspace:* version: link:../logger '@openfn/runtime': specifier: workspace:* version: link:../runtime - '@openfn/runtime-manager': - specifier: workspace:* - version: link:../runtime-manager '@types/koa-logger': specifier: ^3.1.2 version: 3.1.2 @@ -420,7 +518,7 @@ importers: version: 7.4.4 '@types/node': specifier: ^18.15.3 - version: 18.15.3 + version: 18.15.13 '@types/yargs': specifier: ^17.0.12 version: 17.0.24 @@ -441,7 +539,7 @@ importers: version: 8.1.0 ts-node: specifier: ^10.9.1 - version: 10.9.1(@types/node@18.15.3)(typescript@4.6.4) + version: 10.9.1(@types/node@18.15.13)(typescript@4.6.4) tslib: specifier: ^2.4.0 version: 2.4.0 @@ -455,104 +553,6 @@ importers: specifier: ^17.6.2 version: 17.7.2 - packages/runtime: - dependencies: - '@openfn/logger': - specifier: workspace:* - version: link:../logger - fast-safe-stringify: - specifier: ^2.1.1 - version: 2.1.1 - semver: - specifier: ^7.5.4 - version: 7.5.4 - devDependencies: - '@openfn/language-common': - specifier: 2.0.0-rc3 - version: 2.0.0-rc3 - '@types/mock-fs': - specifier: ^4.13.1 - version: 4.13.1 - '@types/node': - specifier: ^18.15.13 - version: 18.15.13 - '@types/semver': - specifier: ^7.5.0 - version: 7.5.0 - ava: - specifier: 5.3.1 - version: 5.3.1 - mock-fs: - specifier: ^5.1.4 - version: 5.1.4 - ts-node: - specifier: ^10.9.1 - version: 10.9.1(@types/node@18.15.13)(typescript@5.1.6) - tslib: - specifier: ^2.4.0 - version: 2.4.0 - tsup: - specifier: ^7.2.0 - version: 7.2.0(ts-node@10.9.1)(typescript@5.1.6) - typescript: - specifier: ^5.1.6 - version: 5.1.6 - - packages/runtime-manager: - dependencies: - '@openfn/compiler': - specifier: workspace:* - version: link:../compiler - '@openfn/language-common': - specifier: 2.0.0-rc3 - version: 2.0.0-rc3 - '@openfn/logger': - specifier: workspace:* - version: link:../logger - '@openfn/runtime': - specifier: workspace:* - version: link:../runtime - koa: - specifier: ^2.13.4 - version: 2.13.4 - workerpool: - specifier: ^6.2.1 - version: 6.2.1 - devDependencies: - '@types/koa': - specifier: ^2.13.5 - version: 2.13.5 - '@types/node': - specifier: ^18.15.13 - version: 18.15.13 - '@types/nodemon': - specifier: ^1.19.2 - version: 1.19.2 - '@types/workerpool': - specifier: ^6.1.0 - version: 6.1.0 - ava: - specifier: 5.3.1 - version: 5.3.1 - nodemon: - specifier: ^2.0.19 - version: 2.0.19 - ts-node: - specifier: ^10.9.1 - version: 10.9.1(@types/node@18.15.13)(typescript@5.1.6) - tslib: - specifier: ^2.4.0 - version: 2.4.0 - tsm: - specifier: ^2.2.2 - version: 2.2.2 - tsup: - specifier: ^7.2.0 - version: 7.2.0(ts-node@10.9.1)(typescript@5.1.6) - typescript: - specifier: ^5.1.6 - version: 5.1.6 - packages: /@babel/code-frame@7.18.6: @@ -1495,7 +1495,6 @@ packages: /@openfn/language-common@2.0.0-rc3: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} - bundledDependencies: [] /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} @@ -6208,7 +6207,7 @@ packages: optional: true dependencies: lilconfig: 2.1.0 - ts-node: 10.9.1(@types/node@18.15.3)(typescript@4.6.4) + ts-node: 10.9.1(@types/node@18.15.13)(typescript@4.6.4) yaml: 1.10.2 dev: true @@ -7392,7 +7391,7 @@ packages: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - /ts-node@10.9.1(@types/node@18.15.13)(typescript@5.1.6): + /ts-node@10.9.1(@types/node@18.15.13)(typescript@4.6.4): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -7412,18 +7411,18 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 '@types/node': 18.15.13 - acorn: 8.8.1 + acorn: 8.10.0 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 5.1.6 + typescript: 4.6.4 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true - /ts-node@10.9.1(@types/node@18.15.3)(typescript@4.6.4): + /ts-node@10.9.1(@types/node@18.15.13)(typescript@5.1.6): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -7442,14 +7441,14 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 - '@types/node': 18.15.3 - acorn: 8.8.1 + '@types/node': 18.15.13 + acorn: 8.10.0 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 make-error: 1.3.6 - typescript: 4.6.4 + typescript: 5.1.6 v8-compile-cache-lib: 3.0.1 yn: 3.1.1 dev: true From 33d9499b482b1d3f829ca50c67518024c4e5df5d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 29 Sep 2023 14:49:25 +0100 Subject: [PATCH 101/232] worker: several fixes to stability and visibility of worker --- packages/ws-worker/README.md | 66 ++++++++++++++----- packages/ws-worker/src/api/execute.ts | 3 +- packages/ws-worker/src/api/start-attempt.ts | 14 +++- packages/ws-worker/src/api/workloop.ts | 53 +++++++++++---- packages/ws-worker/src/events.ts | 9 +-- .../ws-worker/src/mock/lightning/api-dev.ts | 7 +- .../src/mock/lightning/api-sockets.ts | 1 + .../src/mock/lightning/socket-server.ts | 5 +- packages/ws-worker/src/server.ts | 26 ++++---- packages/ws-worker/src/start.ts | 21 ++++-- .../ws-worker/src/util/try-with-backoff.ts | 13 ++-- packages/ws-worker/src/util/worker-token.ts | 15 ++++- packages/ws-worker/test/server.test.ts | 3 + 13 files changed, 174 insertions(+), 62 deletions(-) diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index 84df169dd..66ea922db 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -13,37 +13,37 @@ This package contains: The mock services allow lightweight and controlled testing of the interfaces between them. -## Architecture +## Getting started -Lightning is expected to maintain a queue of attempts. The Worker pulls those attempts from the queue, via websocket, and sends them off to the Engine for execution. +There are several components you may want to run to get started. -While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements. - -The ws-worker server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). +If you're running a local lightning server, remember to set OPENFN_WORKER_SECRET. -## Dev server +### WS Server -You can start a dev server by running: +To start a `ws-socket` server, run: ``` -pnpm start:watch +pnpm start ``` -This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `OPENFN_RTE_REPO_DIR` or `/tmp/openfn/repo`. +This will try and connect to lightning at `localhost:4000`. Use `-l mock` to connect to the default mock server from this repo (you'll need to start the lightning server), or pass your own url to a lightning isntance. -To connect to a lightning instance, pass the `-l` flag. Use `-l mock` to connect to the default mock server from this repo, or pass your own url. +You can start a dev server (which rebuilds on save) by running: -## Lightning Mock +``` +pnpm start:watch +``` -The key API is in `src/mock/lightning/api-socket/ts`. The `createSocketAPI` function hooks up websockets and binds events to event handlers. It's supposed to be quite declarative so you can track the API quite easily. +This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `OPENFN_RTE_REPO_DIR` or `/tmp/openfn/repo`. -See `src/events.ts` for a typings of the expected event names, payloads and replies. +To connect to a lightning instance, pass the `-l` flag. -Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. +### Lightning Mock You can start a Lightning mock server with: -```s +``` pnpm start:lightning ``` @@ -54,3 +54,39 @@ You can add an attempt (`{ jobs, triggers, edges }`) to the queue with: ``` curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" ``` + +Here's an example attempt: + +``` +{ + "id": "my-attempt, + "triggers": [], + "edges": [], + "jobs": [ + { + "id": "job1", + "state": { "data": { "done": true } }, + "adaptor": "@openfn/language-common@1.7.7", + "body": "{ \"result\": 42 }" + } + ] +} +``` + +## Architecture + +Lightning is expected to maintain a queue of attempts. The Worker pulls those attempts from the queue, via websocket, and sends them off to the Engine for execution. + +While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements. + +The ws-worker server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). + +### ws-worker + +### Lightning Mock + +The key API is in `src/mock/lightning/api-socket/ts`. The `createSocketAPI` function hooks up websockets and binds events to event handlers. It's supposed to be quite declarative so you can track the API quite easily. + +See `src/events.ts` for a typings of the expected event names, payloads and replies. + +Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index fe3bf24c9..2f34ead93 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -17,7 +17,6 @@ import { ATTEMPT_LOG_PAYLOAD, ATTEMPT_START, ATTEMPT_START_PAYLOAD, - GET_ATTEMPT, GET_CREDENTIAL, GET_DATACLIP, RUN_COMPLETE, @@ -25,7 +24,7 @@ import { RUN_START, RUN_START_PAYLOAD, } from '../events'; -import { Attempt, Channel } from '../types'; +import { Channel } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; import { getWithReply } from '../util'; diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index 0b0a627f8..910589f01 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -1,10 +1,13 @@ import phx from 'phoenix-channels'; + import convertAttempt from '../util/convert-attempt'; import { getWithReply } from '../util'; import { Attempt, Channel } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; import { GET_ATTEMPT } from '../events'; +import type { Logger } from '@openfn/logger'; + // TODO what happens if this channel join fails? // Lightning could vanish, channel could error on its side, or auth could be wrong // We don't have a good feedback mechanism yet - attempts:queue is the only channel @@ -13,18 +16,25 @@ import { GET_ATTEMPT } from '../events'; const joinAttemptChannel = ( socket: phx.Socket, token: string, - attemptId: string + attemptId: string, + logger: Logger ) => { return new Promise<{ channel: Channel; plan: ExecutionPlan }>( (resolve, reject) => { - const channel = socket.channel(`attempt:${attemptId}`, { token }); + // TODO use proper logger + const channelName = `attempt:${attemptId}`; + logger.debug('connecting to ', channelName); + const channel = socket.channel(channelName, { token }); channel .join() .receive('ok', async () => { + logger.success(`connected to ${channelName}`); const plan = await loadAttempt(channel); + logger.debug('converted attempt as exuction plan:', plan); resolve({ channel, plan }); }) .receive('error', (err) => { + logger.error(`error connecting to ${channelName}`, err); reject(err); }); } diff --git a/packages/ws-worker/src/api/workloop.ts b/packages/ws-worker/src/api/workloop.ts index 2928dd0e8..3e1b74605 100644 --- a/packages/ws-worker/src/api/workloop.ts +++ b/packages/ws-worker/src/api/workloop.ts @@ -1,36 +1,61 @@ -import { CLAIM } from '../events'; -import { CancelablePromise } from '../types'; -import tryWithBackoff from '../util/try-with-backoff'; +import type { Logger } from '@openfn/logger'; +import { CLAIM, CLAIM_ATTEMPT, CLAIM_PAYLOAD, CLAIM_REPLY } from '../events'; +import tryWithBackoff, { Options } from '../util/try-with-backoff'; + +import type { CancelablePromise, Channel } from '../types'; // TODO this needs to return some kind of cancel function -const startWorkloop = (channel, execute, delay = 100) => { +const startWorkloop = ( + channel: Channel, + execute: (attempt: CLAIM_ATTEMPT) => void, + logger: Logger, + options: Partial> = {} +) => { let promise: CancelablePromise; let cancelled = false; const request = () => { return new Promise((resolve, reject) => { - channel.push(CLAIM).receive('ok', (attempts) => { - if (!attempts?.length) { - // throw to backoff and try again - return reject(new Error('backoff')); - } + logger.debug('pull claim'); + channel + .push(CLAIM, { demand: 1 }) + .receive('ok', (attempts: CLAIM_REPLY) => { + // TODO what if we get here after we've been cancelled? + // the events have already been claimed... - attempts.forEach((attempt) => { - execute(attempt); - resolve(); + if (!attempts?.length) { + logger.debug('no attempts, backing off'); + // throw to backoff and try again + return reject(new Error('backoff')); + } + + attempts.forEach((attempt) => { + logger.debug('starting attempt', attempt.id); + execute(attempt); + resolve(); + }); }); - }); }); }; const workLoop = () => { if (!cancelled) { - promise = tryWithBackoff(request, { timeout: delay }); + promise = tryWithBackoff(request, { + timeout: options.delay, + maxBackoff: options.maxBackoff, + }); + // TODO this needs more unit tests I think + promise.then(() => { + if (!cancelled) { + workLoop(); + } + }); } }; workLoop(); return () => { + logger.debug('cancelling workloop'); cancelled = true; promise.cancel(); }; diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 1d9f038dc..51407d7cc 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -4,13 +4,15 @@ import { JSONLog } from '@openfn/logger'; export const CLAIM = 'attempt:claim'; // this is the lightning payload +// TODO why are the types in all caps...? export type CLAIM_PAYLOAD = { demand?: number }; -export type CLAIM_REPLY = Array<{ id: string; token?: string }>; +export type CLAIM_REPLY = Array; +export type CLAIM_ATTEMPT = { id: string; token: string }; export const GET_ATTEMPT = 'fetch:attempt'; export type GET_ATTEMPT_PAYLOAD = void; // no payload // This is basically the attempt, which needs defining properly -export type GET_ATTEMPT_REPLY = Uint8Array; // represents a json string Attempt +export type GET_ATTEMPT_REPLY = Attempt; // TODO type Attempt = { @@ -27,8 +29,7 @@ export type GET_CREDENTIAL_REPLY = {}; export const GET_DATACLIP = 'fetch:dataclip'; export type GET_DATACLIP_PAYLOAD = { id: string }; -// dataclip in-line, no wrapper, arbitrary data -export type GET_DATACLIP_REPLY = {}; +export type GET_DATACLIP_REPLY = Uint8Array; // represents a json string Attempt export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp export type ATTEMPT_START_PAYLOAD = void; // no payload diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts index 683cb9122..4f503d316 100644 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ b/packages/ws-worker/src/mock/lightning/api-dev.ts @@ -16,7 +16,7 @@ type LightningEvents = 'log' | 'attempt-complete'; export type DevApp = Koa & { addCredential(id: string, cred: Credential): void; waitForResult(attemptId: string): Promise; - enqueueAttempt(attempt: Attempt, rtmId: string): void; + enqueueAttempt(attempt: Attempt): void; reset(): void; getQueueLength(): number; getResult(attemptId: string): any; @@ -109,6 +109,11 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { router.post('/attempt', (ctx) => { const attempt = ctx.request.body; + if (!attempt) { + ctx.response.status = 400; + return; + } + logger.info('Adding new attempt to queue:', attempt.id); logger.debug(attempt); diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index a4f0904ce..819dba10f 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -68,6 +68,7 @@ const createSocketAPI = ( }); const startAttempt = (attemptId: string) => { + console.log(`START attempt:${attemptId}`); // mark the attempt as started on the server state.pending[attemptId] = { status: 'started', diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts index d15f4e27d..f0f5f6723 100644 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ b/packages/ws-worker/src/mock/lightning/socket-server.ts @@ -70,7 +70,10 @@ function createServer({ onMessage = () => {}, }: CreateServerOptions) { logger?.info('pheonix mock websocket server listening on', port); - const channels: Record> = {}; + const channels: Record> = { + // create a stub listener for pheonix to prevent errors + phoenix: new Set([() => null]), + }; const wsServer = server || diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 2b0891c61..30b466d1f 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -9,9 +9,11 @@ import startWorkloop from './api/workloop'; import { execute } from './api/execute'; import joinAttemptChannel from './api/start-attempt'; import connectToLightning from './api/connect'; +import { CLAIM_ATTEMPT } from './events'; type ServerOptions = { - backoff?: number; + backoff?: number; // what is this? + maxBackoff?: number; maxWorkflows?: number; port?: number; lightning?: string; // url to lightning instance @@ -45,26 +47,28 @@ function createServer(engine: any, options: ServerOptions = {}) { logger.info('Closing server'); }; - type StartAttemptArgs = { - id: string; - token: string; - }; - if (options.lightning) { - logger.log('Starting work loop at', options.lightning); + logger.debug('Connecting to Lightning at', options.lightning); + // TODO this is too hard to unit test, need to pull it out connectToLightning(options.lightning, engine.id, options.secret!).then( ({ socket, channel }) => { - const startAttempt = async ({ id, token }: StartAttemptArgs) => { + logger.success('Connected to Lightning at', options.lightning); + + const startAttempt = async ({ id, token }: CLAIM_ATTEMPT) => { const { channel: attemptChannel, plan } = await joinAttemptChannel( socket, token, - id + id, + logger ); execute(attemptChannel, engine, plan); }; - // TODO maybe pull this logic out so we can test it? - startWorkloop(channel, startAttempt); + logger.info('Starting workloop'); + // TODO maybe namespace the workloop logger differently? It's a bit annoying + startWorkloop(channel, startAttempt, logger, { + maxBackoff: options.maxBackoff, + }); // debug/unit test API to run a workflow // TODO Only loads in dev mode? diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index eb15955c5..290334718 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -15,8 +15,6 @@ type Args = { secret?: string; }; -const logger = createLogger('SRV', { level: 'info' }); - const args = yargs(hideBin(process.argv)) .command('server', 'Start a ws-worker server') .option('port', { @@ -25,6 +23,8 @@ const args = yargs(hideBin(process.argv)) type: 'number', default: 2222, }) + // TODO maybe this is positional and required? + // frees up -l for the log .option('lightning', { alias: 'l', description: @@ -36,18 +36,27 @@ const args = yargs(hideBin(process.argv)) }) .option('secret', { alias: 's', - description: 'Worker secret (comes from WORKER_SECRET by default)', + description: 'Worker secret (comes from OPENFN_WORKER_SECRET by default)', + }) + .option('log', { + description: 'Worker secret (comes from OPENFN_WORKER_SECRET by default)', + default: 'info', + type: 'string', }) .parse() as Args; +const logger = createLogger('SRV', { level: args.log }); + if (args.lightning === 'mock') { args.lightning = 'ws://localhost:8888/api'; } else if (!args.secret) { - if (!process.env.WORKER_SECRET) { - console.error('WORKER_SECRET is not set'); + const { OPENFN_WORKER_SECRET } = process.env; + if (!OPENFN_WORKER_SECRET) { + logger.error('OPENFN_WORKER_SECRET is not set'); process.exit(1); } - args.secret = process.env.WORKER_SECRET; + + args.secret = OPENFN_WORKER_SECRET; } // TODO the engine needs to take callbacks to load credential, and load state diff --git a/packages/ws-worker/src/util/try-with-backoff.ts b/packages/ws-worker/src/util/try-with-backoff.ts index 6caa80eca..225b74bc5 100644 --- a/packages/ws-worker/src/util/try-with-backoff.ts +++ b/packages/ws-worker/src/util/try-with-backoff.ts @@ -1,25 +1,30 @@ import { CancelablePromise } from '../types'; -type Options = { +export type Options = { attempts?: number; maxAttempts?: number; maxBackoff?: number; + + // these are provided internally timeout?: number; isCancelled?: () => boolean; }; -const MAX_BACKOFF = 1000 * 60; +const MAX_BACKOFF = 1000 * 30; // This function will try and call its first argument every {opts.timeout|100}ms // If the function throws, it will "backoff" and try again a little later // Right now it's a bit of a sketch, but it sort of works! const tryWithBackoff = (fn: any, opts: Options = {}): CancelablePromise => { if (!opts.timeout) { - opts.timeout = 100; + opts.timeout = 100; // TODO take this as minBackoff or initialBackoff or something } if (!opts.attempts) { opts.attempts = 1; } + if (!opts.maxBackoff) { + opts.maxBackoff = MAX_BACKOFF; + } let { timeout, attempts, maxAttempts } = opts; timeout = timeout; attempts = attempts; @@ -53,7 +58,7 @@ const tryWithBackoff = (fn: any, opts: Options = {}): CancelablePromise => { const nextOpts = { maxAttempts, attempts: attempts + 1, - timeout: Math.min(MAX_BACKOFF, timeout * 1.2), + timeout: Math.min(opts.maxBackoff!, timeout * 1.2), isCancelled: opts.isCancelled, }; diff --git a/packages/ws-worker/src/util/worker-token.ts b/packages/ws-worker/src/util/worker-token.ts index fe13e6ca6..eba9bb035 100644 --- a/packages/ws-worker/src/util/worker-token.ts +++ b/packages/ws-worker/src/util/worker-token.ts @@ -3,7 +3,18 @@ import * as jose from 'jose'; const alg = 'HS256'; const generateWorkerToken = async (secret: string, workerId: string) => { - const encodedSecret = new TextEncoder().encode(secret); + if (!secret) { + // TODO use proper logger + // TODO is this even necessary? + console.warn(); + console.warn('WARNING: Worker Secret not provided!'); + console.warn( + 'This worker will attempt to connect to Lightning with default secret' + ); + console.warn(); + } + + const encodedSecret = new TextEncoder().encode(secret || ''); const claims = { worker_id: workerId, @@ -14,8 +25,8 @@ const generateWorkerToken = async (secret: string, workerId: string) => { .setIssuedAt() .setIssuer('urn:example:issuer') .setAudience('urn:example:audience') - // .setExpirationTime('2h') .sign(encodedSecret); + // .setExpirationTime('2h') // ?? return jwt; }; diff --git a/packages/ws-worker/test/server.test.ts b/packages/ws-worker/test/server.test.ts index aa67b5115..ced018d82 100644 --- a/packages/ws-worker/test/server.test.ts +++ b/packages/ws-worker/test/server.test.ts @@ -34,6 +34,9 @@ test.skip('healthcheck', async (t) => { t.is(body, 'OK'); }); +test.todo('do something if we fail to connect to lightning'); +test.todo("don't explode if no lightning endpoint is set (or maybe do?)"); + // TODO this isn't testing anything now, see test/api/connect.test.ts // Not a very thorough test // test.only('connects to lightning', async (t) => { From 86ffb4e277af401f072849d087126619c87aa67a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 29 Sep 2023 15:48:48 +0100 Subject: [PATCH 102/232] worker: refactor of execute, plus fixes --- packages/ws-worker/src/api/execute.ts | 129 +++++++++++------- packages/ws-worker/src/api/start-attempt.ts | 2 +- .../src/mock/lightning/api-sockets.ts | 3 +- packages/ws-worker/src/server.ts | 2 +- packages/ws-worker/test/api/execute.test.ts | 64 ++++++--- .../ws-worker/test/api/start-attempt.test.ts | 12 +- packages/ws-worker/test/api/workloop.test.ts | 15 +- 7 files changed, 149 insertions(+), 78 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 2f34ead93..e344394b6 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -1,14 +1,4 @@ -// TODO not crazy about this file name -// This is the module responsible for interfacing between the Lightning websocket -// and the rutnime engine -// It's the actual meat and potatoes of the implementation -// You can almost read this as a binding function and a bunch of handlers -// it isn't an actual worker, but a BRIDGE between a worker and lightning import crypto from 'node:crypto'; -import { JSONLog } from '@openfn/logger'; - -// this managers the worker -//i would like functions to be testable, and I'd like the logic to be readable import { ATTEMPT_COMPLETE, @@ -25,8 +15,14 @@ import { RUN_START_PAYLOAD, } from '../events'; import { Channel } from '../types'; -import { ExecutionPlan } from '@openfn/runtime'; -import { getWithReply } from '../util'; +import { getWithReply, stringify } from '../util'; + +import type { ExecutionPlan } from '@openfn/runtime'; +import type { JSONLog, Logger } from '@openfn/logger'; +import { + WorkflowCompleteEvent, + WorkflowStartEvent, +} from '../mock/runtime-engine'; const enc = new TextDecoder('utf-8'); @@ -40,9 +36,21 @@ export type AttemptState = { // TODO status? }; +type Context = { + channel: Channel; + state: AttemptState; + logger: Logger; + onComplete: (result: any) => void; +}; + // pass a web socket connected to the attempt channel // this thing will do all the work -export function execute(channel: Channel, engine, plan: ExecutionPlan) { +export function execute( + channel: Channel, + engine: any, // TODO typing! + logger: Logger, + plan: ExecutionPlan +) { return new Promise((resolve) => { // TODO add proper logger (maybe channel, rtm and logger comprise a context object) // tracking state for this attempt @@ -50,20 +58,38 @@ export function execute(channel: Channel, engine, plan: ExecutionPlan) { plan, }; - // TODO - // const context = { channel, state, logger } - - engine.listen(plan.id, { - // TODO load event types from runtime-manager - 'workflow-start': (evt: any) => onWorkflowStart(channel), - 'job-start': (evt: any) => onJobStart(channel, state, evt), - 'job-complete': (evt: any) => onJobComplete(channel, state, evt), - log: (evt: any) => onJobLog(channel, state, evt), - 'workflow-complete': (evt: any) => { - onWorkflowComplete(channel, state, evt); - resolve(evt.state); - }, - }); + const context: Context = { channel, state, logger, onComplete: resolve }; + + type EventHandler = (context: any, event: any) => void; + + // Utility funciton to + // a) bind an event handler to a event + // b) pass the contexdt object into the hander + // c) log the event + const addEvent = (eventName: string, handler: EventHandler) => { + const wrappedFn = (event: any) => { + logger.info(`${plan.id} :: ${eventName}`); + handler(context, event); + }; + return { + [eventName]: wrappedFn, + }; + }; + + // TODO we should wait for each event to complete before sending the next one + // Eg wait for a large dataclip to upload back to lightning before starting the next job + // should we actually defer exeuction, or just the reporting? + // Does it matter if logs aren't sent back in order? + const listeners = Object.assign( + {}, + addEvent('workflow-start', onWorkflowStart), + addEvent('job-start', onJobStart), + addEvent('job-complete', onJobComplete), + addEvent('log', onJobLog), + // This will also resolve the promise + addEvent('workflow-complete', onWorkflowComplete) + ); + engine.listen(plan.id, listeners); const resolvers = { state: (id: string) => loadState(channel, id), @@ -76,58 +102,57 @@ export function execute(channel: Channel, engine, plan: ExecutionPlan) { // TODO maybe move all event handlers into api/events/* -export function onJobStart( - channel: Channel, - state: AttemptState, - jobId: string -) { +export function onJobStart({ channel, state }: Context, event: any) { // generate a run id and write it to state state.activeRun = crypto.randomUUID(); - state.activeJob = jobId; + state.activeJob = event; channel.push(RUN_START, { - run_id: state.activeJob, - job_id: state.activeJob, + run_id: state.activeJob!, + job_id: state.activeJob!, // input_dataclip_id what about this guy? }); } -export function onJobComplete( - channel: Channel, - state: AttemptState, - evt: any // TODO need to type the engine events nicely -) { +export function onJobComplete({ channel, state }: Context, event: any) { channel.push(RUN_COMPLETE, { run_id: state.activeRun!, job_id: state.activeJob!, - output_dataclip: JSON.stringify(evt.state), + // TODO generate a dataclip id + output_dataclip: stringify(event.state), }); delete state.activeRun; delete state.activeJob; } -export function onWorkflowStart(channel: Channel) { +export function onWorkflowStart( + { channel }: Context, + _event: WorkflowStartEvent +) { channel.push(ATTEMPT_START); } export function onWorkflowComplete( - channel: Channel, - state: AttemptState, - evt: any + { state, channel, onComplete }: Context, + event: WorkflowCompleteEvent ) { - state.result = evt.state; - - channel.push(ATTEMPT_COMPLETE, { - dataclip: evt.state, - }); + state.result = event.state; + + channel + .push(ATTEMPT_COMPLETE, { + dataclip: stringify(event.state), // TODO this should just be dataclip id + }) + .receive('ok', () => { + onComplete(state.result); + }); } -export function onJobLog(channel: Channel, state: AttemptState, log: JSONLog) { +export function onJobLog({ channel, state }: Context, event: JSONLog) { // we basically just forward the log to lightning // but we also need to attach the log id const evt: ATTEMPT_LOG_PAYLOAD = { - ...log, + ...event, attempt_id: state.plan.id!, }; if (state.activeRun) { diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index 910589f01..a33747762 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -30,7 +30,7 @@ const joinAttemptChannel = ( .receive('ok', async () => { logger.success(`connected to ${channelName}`); const plan = await loadAttempt(channel); - logger.debug('converted attempt as exuction plan:', plan); + logger.debug('converted attempt as execution plan:', plan); resolve({ channel, plan }); }) .receive('error', (err) => { diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index 819dba10f..d8dffc228 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -68,7 +68,8 @@ const createSocketAPI = ( }); const startAttempt = (attemptId: string) => { - console.log(`START attempt:${attemptId}`); + logger && logger.debug(`joining channel attempt:${attemptId}`); + // mark the attempt as started on the server state.pending[attemptId] = { status: 'started', diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 30b466d1f..6c0fe5cb6 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -61,7 +61,7 @@ function createServer(engine: any, options: ServerOptions = {}) { id, logger ); - execute(attemptChannel, engine, plan); + execute(attemptChannel, engine, logger, plan); }; logger.info('Starting workloop'); diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 2cbff53c4..498ca573b 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { JSONLog } from '@openfn/logger'; +import { JSONLog, createMockLogger } from '@openfn/logger'; import { RUN_START, @@ -39,7 +39,7 @@ test('jobStart should set a run id and active job on state', async (t) => { const channel = mockChannel({}); - onJobStart(channel, state, jobId); + onJobStart({ channel, state }, jobId); t.is(state.activeJob, jobId); t.truthy(state.activeRun); @@ -63,7 +63,7 @@ test('jobStart should send a run:start event', async (t) => { }, }); - onJobStart(channel, state, jobId); + onJobStart({ channel, state }, jobId); }); }); @@ -79,7 +79,8 @@ test('jobComplete should clear the run id and active job on state', async (t) => const channel = mockChannel({}); - onJobComplete(channel, state, { state: { x: 10 } }); + const event = { state: { x: 10 } }; + onJobComplete({ channel, state }, event); t.falsy(state.activeJob); t.falsy(state.activeRun); @@ -107,7 +108,8 @@ test('jobComplete should send a run:complete event', async (t) => { }, }); - onJobComplete(channel, state, { state: result }); + const event = { state: result }; + onJobComplete({ channel, state }, event); }); }); @@ -139,7 +141,7 @@ test('jobLog should should send a log event outside a run', async (t) => { }, }); - onJobLog(channel, state, log); + onJobLog({ channel, state }, log); }); }); @@ -172,7 +174,7 @@ test('jobLog should should send a log event inside a run', async (t) => { }, }); - onJobLog(channel, state, log); + onJobLog({ channel, state }, log); }); }); @@ -186,7 +188,7 @@ test('workflowStart should send an empty attempt:start event', async (t) => { }, }); - onWorkflowStart(channel); + onWorkflowStart({ channel }); }); }); @@ -198,14 +200,40 @@ test('workflowComplete should send an attempt:complete event', async (t) => { const channel = mockChannel({ [ATTEMPT_COMPLETE]: (evt) => { - t.deepEqual(evt.dataclip, result); + t.deepEqual(evt.dataclip, JSON.stringify(result)); t.deepEqual(state.result, result); done(); }, }); - onWorkflowComplete(channel, state, { state: result }); + const event = { state: result }; + + const context = { channel, state, onComplete: () => {} }; + onWorkflowComplete(context, event); + }); +}); + +test('workflowComplete should call onComplete with state', async (t) => { + return new Promise((done) => { + const state = {} as AttemptState; + + const result = { answer: 42 }; + + const channel = mockChannel(); + + const context = { + channel, + state, + onComplete: (finalState) => { + t.deepEqual(result, finalState); + done(); + }, + }; + + const event = { state: result }; + + onWorkflowComplete(context, event); }); }); @@ -238,6 +266,7 @@ test('loadCredential should fetch a credential', async (t) => { test('execute should return the final result', async (t) => { const channel = mockChannel(); const engine = createMockRTE(); + const logger = createMockLogger(); const plan = { id: 'a', @@ -248,13 +277,14 @@ test('execute should return the final result', async (t) => { ], }; - const result = await execute(channel, engine, plan); + const result = await execute(channel, engine, logger, plan); t.deepEqual(result, { done: true }); }); // TODO this is more of an engine test really, but worth having I suppose test('execute should lazy-load a credential', async (t) => { + const logger = createMockLogger(); let didCallCredentials = false; const channel = mockChannel({ @@ -276,13 +306,14 @@ test('execute should lazy-load a credential', async (t) => { ], }; - await execute(channel, engine, plan); + await execute(channel, engine, logger, plan); t.true(didCallCredentials); }); // TODO this is more of an engine test really, but worth having I suppose test('execute should lazy-load initial state', async (t) => { + const logger = createMockLogger(); let didCallState = false; const channel = mockChannel({ @@ -304,16 +335,17 @@ test('execute should lazy-load initial state', async (t) => { ], }; - await execute(channel, engine, plan); + await execute(channel, engine, logger, plan); t.true(didCallState); }); test('execute should call all events on the socket', async (t) => { - const events = {}; - + const logger = createMockLogger(); const engine = createMockRTE(); + const events = {}; + const toEventMap = (obj, evt: string) => { obj[evt] = (e) => { events[evt] = e || true; @@ -346,7 +378,7 @@ test('execute should call all events on the socket', async (t) => { ], }; - await execute(channel, engine, plan); + await execute(channel, engine, logger, plan); // check result is what we expect diff --git a/packages/ws-worker/test/api/start-attempt.test.ts b/packages/ws-worker/test/api/start-attempt.test.ts index 51c447a60..b79f9160c 100644 --- a/packages/ws-worker/test/api/start-attempt.test.ts +++ b/packages/ws-worker/test/api/start-attempt.test.ts @@ -4,6 +4,7 @@ import joinAttemptChannel from '../../src/api/start-attempt'; import { GET_ATTEMPT } from '../../src/events'; import { loadAttempt } from '../../src/api/start-attempt'; import { attempts } from '../mock/data'; +import { createMockLogger } from '@openfn/logger'; test('loadAttempt should get the attempt body', async (t) => { const attempt = attempts['attempt-1']; @@ -42,6 +43,7 @@ test('loadAttempt should return an execution plan', async (t) => { }); test('should join an attempt channel with a token', async (t) => { + const logger = createMockLogger(); const socket = mockSocket('www', { 'attempt:a': mockChannel({ // Note that the validation logic is all handled here @@ -52,13 +54,19 @@ test('should join an attempt channel with a token', async (t) => { }), }); - const { channel, plan } = await joinAttemptChannel(socket, 'x.y.z', 'a'); + const { channel, plan } = await joinAttemptChannel( + socket, + 'x.y.z', + 'a', + logger + ); t.truthy(channel); t.deepEqual(plan, { id: 'a', jobs: [] }); }); test('should fail to join an attempt channel with an invalid token', async (t) => { + const logger = createMockLogger(); const socket = mockSocket('www', { 'attempt:a': mockChannel({ // Note that the validation logic is all handled here @@ -72,7 +80,7 @@ test('should fail to join an attempt channel with an invalid token', async (t) = try { // ts-ignore - await joinAttemptChannel(socket, 'x.y.z', 'a'); + await joinAttemptChannel(socket, 'x.y.z', 'a', logger); } catch (e) { // the error here is whatever is passed as the response to the receive-error event t.is(e, 'invalid-token'); diff --git a/packages/ws-worker/test/api/workloop.test.ts b/packages/ws-worker/test/api/workloop.test.ts index 3f802e3f5..d6c944278 100644 --- a/packages/ws-worker/test/api/workloop.test.ts +++ b/packages/ws-worker/test/api/workloop.test.ts @@ -5,9 +5,13 @@ import { sleep } from '../util'; import { mockChannel } from '../../src/mock/sockets'; import startWorkloop from '../../src/api/workloop'; import { CLAIM } from '../../src/events'; +import { createMockLogger } from '@openfn/logger'; +import { execute } from '../../src/api/execute'; let cancel; +const logger = createMockLogger(); + test.afterEach(() => { cancel?.(); // cancel any workloops }); @@ -22,7 +26,7 @@ test('workloop can be cancelled', async (t) => { }, }); - cancel = startWorkloop(channel, () => {}, 1); + cancel = startWorkloop(channel, () => {}, logger, { timeout: 1 }); await sleep(100); // A quirk of how cancel works is that the loop will be called a few times @@ -38,7 +42,7 @@ test('workloop sends the attempts:claim event', (t) => { done(); }, }); - cancel = startWorkloop(channel, () => {}); + cancel = startWorkloop(channel, () => {}, logger); }); }); @@ -55,7 +59,7 @@ test('workloop sends the attempts:claim event several times ', (t) => { } }, }); - cancel = startWorkloop(channel, () => {}); + cancel = startWorkloop(channel, () => {}, logger); }); }); @@ -68,10 +72,11 @@ test('workloop calls execute if attempts:claim returns attempts', (t) => { }, }); - cancel = startWorkloop(channel, (attempt) => { + const execute = (attempt) => { t.deepEqual(attempt, { id: 'a' }); t.pass(); done(); - }); + }; + cancel = startWorkloop(channel, execute, logger); }); }); From f0043883798f4c2a00acb051fc39b1b56f90378c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 29 Sep 2023 16:51:59 +0100 Subject: [PATCH 103/232] worker: slightly better error handling if lightning can't be found --- packages/ws-worker/src/server.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 6c0fe5cb6..f82f5ca32 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -50,8 +50,8 @@ function createServer(engine: any, options: ServerOptions = {}) { if (options.lightning) { logger.debug('Connecting to Lightning at', options.lightning); // TODO this is too hard to unit test, need to pull it out - connectToLightning(options.lightning, engine.id, options.secret!).then( - ({ socket, channel }) => { + connectToLightning(options.lightning, engine.id, options.secret!) + .then(({ socket, channel }) => { logger.success('Connected to Lightning at', options.lightning); const startAttempt = async ({ id, token }: CLAIM_ATTEMPT) => { @@ -73,8 +73,15 @@ function createServer(engine: any, options: ServerOptions = {}) { // debug/unit test API to run a workflow // TODO Only loads in dev mode? (app as any).execute = startAttempt; - } - ); + }) + .catch((e) => { + logger.error( + 'CRITICAL ERROR: could not connect to lightning at', + options.lightning + ); + logger.debug(e); + process.exit(1); + }); } else { logger.warn('No lightning URL provided'); } From 567dd249d1a5794dcebd856c8603e948774a0075 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 29 Sep 2023 18:09:38 +0100 Subject: [PATCH 104/232] worker: add integration tests, setup pattern --- .../ws-worker/src/mock/lightning/api-dev.ts | 14 ++++ .../src/mock/lightning/api-sockets.ts | 72 ++++++++++------- .../src/mock/lightning/socket-server.ts | 5 +- packages/ws-worker/src/start.ts | 2 +- packages/ws-worker/test/integration.test.ts | 77 +++++++++++++++---- 5 files changed, 127 insertions(+), 43 deletions(-) diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts index 4f503d316..fa7a33c58 100644 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ b/packages/ws-worker/src/mock/lightning/api-dev.ts @@ -99,6 +99,20 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.once = (event: LightningEvents, fn: (evt: any) => void) => { state.events.once(event, fn); }; + + app.onSocketEvent = ( + event: LightningEvents, + attemptId: string, + fn: (evt: any) => void + ) => { + function handler(e: any) { + if (e.attemptId && e.attemptId === attemptId) { + state.events.removeListener(event, handler); + fn(e); + } + } + state.events.addListener(event, handler); + }; }; // Set up some rest endpoints diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index d8dffc228..9401e0d2e 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -33,6 +33,13 @@ import { import type { Server } from 'http'; import { stringify } from '../../util'; +// dumb cloning id +// just an idea for unit tests +const clone = (state: ServerState) => { + const { events, ...rest } = state; + return JSON.parse(JSON.stringify(rest)); +}; + const enc = new TextEncoder(); // this new API is websocket based @@ -63,8 +70,16 @@ const createSocketAPI = ( }); wss.registerEvents('attempts:queue', { - [CLAIM]: (ws, event: PhoenixEvent) => - pullClaim(state, ws, event), + [CLAIM]: (ws, event: PhoenixEvent) => { + const results = pullClaim(state, ws, event); + results.forEach((attempt) => { + state.events.emit(CLAIM, { + attemptId: attempt.id, + payload: attempt, + state: clone(state), + }); + }); + }, }); const startAttempt = (attemptId: string) => { @@ -76,27 +91,35 @@ const createSocketAPI = ( logs: [], }; - // TODO do all these need extra auth, or is auth granted - // implicitly by channel membership? - // Right now the socket gets access to all server state - // But this is just a mock - Lightning can impose more restrictions if it wishes + const wrap = ( + handler: ( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent, + attemptId: string + ) => void + ) => { + return (ws: DevSocket, event: PhoenixEvent) => { + handler(state, ws, event, attemptId); + // emit each event and the state after to the event handler, for debug + state.events.emit(event.event, { + attemptId, + payload: event.payload, + state: clone(state), + }); + }; + }; + const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { - // TODO how will I validate the join in my mock? - [GET_ATTEMPT]: (ws, event: PhoenixEvent) => - getAttempt(state, ws, event), - [GET_CREDENTIAL]: (ws, event: PhoenixEvent) => - getCredential(state, ws, event), - [GET_DATACLIP]: (ws, event: PhoenixEvent) => - getDataclip(state, ws, event), - [ATTEMPT_LOG]: (ws, event: PhoenixEvent) => - handleLog(state, ws, event), - [ATTEMPT_COMPLETE]: ( - ws, - event: PhoenixEvent - ) => { - handleAttemptComplete(state, ws, event, attemptId); + [GET_ATTEMPT]: wrap(getAttempt), + [GET_CREDENTIAL]: wrap(getCredential), + [GET_DATACLIP]: wrap(getDataclip), + [ATTEMPT_LOG]: wrap(handleLog), + [ATTEMPT_COMPLETE]: wrap((...args) => { + handleAttemptComplete(...args); unsubscribe(); - }, + }), + // TODO // [RUN_START] // [RUN_COMPLETE] @@ -142,6 +165,7 @@ const createSocketAPI = ( } ws.reply({ ref, topic, payload }); + return payload.response; } function getAttempt( @@ -236,12 +260,6 @@ const createSocketAPI = ( state.pending[attemptId].status = 'complete'; state.results[attemptId] = dataclip; - state.events.emit(ATTEMPT_COMPLETE, { - attemptId: attemptId, - dataclip, - logs: state.pending[attemptId].logs, - }); - ws.reply({ ref, topic, diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts index f0f5f6723..81cac0fa3 100644 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ b/packages/ws-worker/src/mock/lightning/socket-server.ts @@ -69,7 +69,6 @@ function createServer({ logger, onMessage = () => {}, }: CreateServerOptions) { - logger?.info('pheonix mock websocket server listening on', port); const channels: Record> = { // create a stub listener for pheonix to prevent errors phoenix: new Set([() => null]), @@ -81,6 +80,10 @@ function createServer({ port, }); + if (!server) { + logger?.info('pheonix mock websocket server listening on', port); + } + const events = { // testing (TODO shouldn't this be in a specific channel?) ping: (ws: DevSocket, { topic, ref }: PhoenixEvent) => { diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 290334718..07dffbf8f 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -28,7 +28,7 @@ const args = yargs(hideBin(process.argv)) .option('lightning', { alias: 'l', description: - 'Base url to Lightning, eg, http://localhost:1234. Set to "mock" to use the default mock server', + 'Base url to Lightning websocket endpoint, eg, ws://locahost:4000/api. Set to "mock" to use the default mock server', }) .option('repo-dir', { alias: 'd', diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 6c297a994..67ee4b872 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -6,35 +6,84 @@ import test from 'ava'; import createWorkerServer from '../src/server'; import createLightningServer from '../src/mock/lightning'; import createMockRTE from '../src/mock/runtime-engine'; +import * as e from '../src/events'; let lng; let engine; const urls = { engine: 'http://localhost:4567', - lng: 'http://localhost:7654', + lng: 'ws://localhost:7654/api', }; test.before(() => { + // TODO give lightning the same secret and do some validation lng = createLightningServer({ port: 7654 }); engine = createWorkerServer(createMockRTE('engine'), { port: 4567, lightning: urls.lng, + secret: 'abc', }); }); -// Really high level test -test.serial.skip('process an attempt', async (t) => { - lng.enqueueAttempt({ - id: 'a1', - jobs: [ - { - adaptor: '@openfn/language-common@1.0.0', - body: JSON.stringify({ answer: 42 }), - }, - ], - }); +let rollingAttemptId = 0; - const { state } = await lng.waitForResult('a1'); - t.is(state.answer, 42); +const getAttempt = (ext = {}, jobs = {}) => ({ + id: `a${++rollingAttemptId}`, + jobs: jobs || [ + { + adaptor: '@openfn/language-common@1.0.0', + body: JSON.stringify({ answer: 42 }), + }, + ], + ...ext, }); + +// A basic high level integration test to ensure the whole loop works +// This checks the events received by the lightning websocket +test.serial( + 'worker should pull an event from lightning, lightning should receive attempt-complete', + (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + // TODO we should validate the result event here, but it's not quite decided + // I think it should be { attempt_id, dataclip_id } + t.pass('attempt complete event received'); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +// Now run detailed checks of every event +// for each event we can see a copy of the server state +// (if that helps anything?) + +test.serial(`events: lightning should receive a ${e.CLAIM} event`, (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + let didCallEvent = false; + lng.onSocketEvent(e.CLAIM, attempt.id, ({ payload }) => { + const { id, token } = payload; + // Note that the payload here is what will be sent back to the worker + // TODO check there's a token + t.truthy(id); + t.truthy(token); + t.assert(typeof token === 'string'); + + didCallEvent = true; + }); + + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); +}); + +// should run multiple concurrently From 96d637a621117dbed231f555636308af7c1fa0bc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 2 Oct 2023 10:39:28 +0100 Subject: [PATCH 105/232] worker: extra unit tests --- packages/ws-worker/src/mock/runtime-engine.ts | 11 +- .../ws-worker/src/util/convert-attempt.ts | 3 +- packages/ws-worker/test/integration.test.ts | 156 +++++++++++++++++- 3 files changed, 164 insertions(+), 6 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 3027f331d..5d599893c 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -117,8 +117,12 @@ function createMock(serverId?: string) { // Get the job details from lightning // start instantly and emit as it goes dispatch('job-start', { workflowId, jobId, runId }); - - dispatch('log', { workflowId, jobId, message: ['Running job ' + jobId] }); + dispatch('log', { + workflowId, + jobId, + message: ['Running job ' + jobId], + level: 'info', + }); let state = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { @@ -128,8 +132,9 @@ function createMock(serverId?: string) { workflowId, jobId, message: ['Parsing expression as JSON state'], + level: 'info', }); - dispatch('log', { workflowId, jobId, message: [state] }); + dispatch('log', { workflowId, jobId, message: [state], level: 'info' }); } catch (e) { // Do nothing, it's fine } diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index 382b0e2a2..fb93f27f0 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -1,3 +1,4 @@ +import crypto from 'node:crypto'; import type { ExecutionPlan, JobNode, JobNodeID } from '@openfn/runtime'; import { Attempt } from '../types'; @@ -35,7 +36,7 @@ export default (attempt: Attempt): ExecutionPlan => { if (attempt.jobs?.length) { attempt.jobs.forEach((job) => { - const id = job.id || 'trigger'; + const id = job.id || crypto.randomUUID(); nodes[id] = { id, diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 67ee4b872..2a5e135ab 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -28,7 +28,7 @@ test.before(() => { let rollingAttemptId = 0; -const getAttempt = (ext = {}, jobs = {}) => ({ +const getAttempt = (ext = {}, jobs?: any) => ({ id: `a${++rollingAttemptId}`, jobs: jobs || [ { @@ -58,6 +58,8 @@ test.serial( } ); +test.todo(`events: lightning should receive a ${e.ATTEMPT_START} event`); + // Now run detailed checks of every event // for each event we can see a copy of the server state // (if that helps anything?) @@ -86,4 +88,154 @@ test.serial(`events: lightning should receive a ${e.CLAIM} event`, (t) => { }); }); -// should run multiple concurrently +test.serial( + `events: lightning should receive a ${e.GET_ATTEMPT} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + + let didCallEvent = false; + lng.onSocketEvent(e.GET_ATTEMPT, attempt.id, ({ payload }) => { + // This doesn't test that the correct attempt gets sent back + // We'd have to add an event to the engine for that + // (not a bad idea) + didCallEvent = true; + }); + + lng.onSocketEvent(e.GET_ATTEMPT, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +test.serial( + `events: lightning should receive a ${e.GET_CREDENTIAL} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt({}, [ + { + id: 'some-job', + credential: 'a', + adaptor: '@openfn/language-common@1.0.0', + body: JSON.stringify({ answer: 42 }), + }, + ]); + + let didCallEvent = false; + lng.onSocketEvent(e.GET_CREDENTIAL, attempt.id, ({ payload }) => { + // again there's no way to check the right credential was returned + didCallEvent = true; + }); + + lng.onSocketEvent(e.GET_CREDENTIAL, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +test.serial.skip( + `events: lightning should receive a ${e.GET_DATACLIP} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt({ + dataclip_id: 'abc', // TODO this isn't implemented yet + }); + + let didCallEvent = false; + lng.onSocketEvent(e.GET_DATACLIP, attempt.id, ({ payload }) => { + // again there's no way to check the right credential was returned + didCallEvent = true; + }); + + lng.onSocketEvent(e.GET_DATACLIP, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +// TODO not implemented yet +test.serial.skip( + `events: lightning should receive a ${e.RUN_START} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + + let didCallEvent = false; + lng.onSocketEvent(e.RUN_START, attempt.id, ({ payload }) => { + // TODO what can we test here? + didCallEvent = true; + }); + + lng.onSocketEvent(e.RUN_START, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +test.serial( + `events: lightning should receive a ${e.ATTEMPT_LOG} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + + let didCallEvent = false; + + // The mock runtime will put out a default log + lng.onSocketEvent(e.ATTEMPT_LOG, attempt.id, ({ payload }) => { + const log = payload; + + t.is(log.level, 'info'); + t.truthy(log.attempt_id); + t.truthy(log.run_id); + t.truthy(log.message); + t.assert(log.message[0].startsWith('Running job')); + + didCallEvent = true; + }); + + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + t.true(didCallEvent); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +test.todo(`events: lightning should receive a ${e.RUN_COMPLETE} event`); + +// This is well tested elsewhere but including here for completeness +test.serial( + `events: lightning should receive a ${e.ATTEMPT_COMPLETE} event`, + (t) => { + return new Promise((done) => { + const attempt = getAttempt(); + + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { + t.pass('called attempt:complete'); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); + +test.todo(`should run multiple attempts`); From d233d76574d55c827ee67b01516c628ad83f1924 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 2 Oct 2023 11:10:00 +0100 Subject: [PATCH 106/232] worker: fix test --- packages/ws-worker/src/mock/lightning/api-dev.ts | 7 ++++--- packages/ws-worker/test/api/execute.test.ts | 4 ++-- packages/ws-worker/test/api/start-attempt.test.ts | 3 +-- packages/ws-worker/test/mock/lightning.test.ts | 12 ++++-------- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts index fa7a33c58..78627e5bb 100644 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ b/packages/ws-worker/src/mock/lightning/api-dev.ts @@ -9,7 +9,7 @@ import crypto from 'node:crypto'; import { Attempt } from '../../types'; import { ServerState } from './server'; -import { ATTEMPT_COMPLETE } from '../../events'; +import { ATTEMPT_COMPLETE, ATTEMPT_COMPLETE_PAYLOAD } from '../../events'; type LightningEvents = 'log' | 'attempt-complete'; @@ -58,14 +58,15 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.waitForResult = (attemptId: string) => { return new Promise((resolve) => { const handler = (evt: { + payload: ATTEMPT_COMPLETE_PAYLOAD; attemptId: string; + _state: ServerState; dataclip: any; - logs: JSONLog[]; }) => { if (evt.attemptId === attemptId) { state.events.removeListener(ATTEMPT_COMPLETE, handler); - resolve(evt); + resolve(evt.payload); } }; state.events.addListener(ATTEMPT_COMPLETE, handler); diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 498ca573b..c20cf3278 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -355,8 +355,8 @@ test('execute should call all events on the socket', async (t) => { const allEvents = [ // Note that these are listed in order but order isn not tested - // GET_CREDENTIAL, // TODO not implementated yet - // GET_DATACLIP, // TODO not implementated yet + GET_CREDENTIAL, + // GET_DATACLIP, // TODO not really implemented properly yet ATTEMPT_START, RUN_START, ATTEMPT_LOG, diff --git a/packages/ws-worker/test/api/start-attempt.test.ts b/packages/ws-worker/test/api/start-attempt.test.ts index b79f9160c..914ccfbaa 100644 --- a/packages/ws-worker/test/api/start-attempt.test.ts +++ b/packages/ws-worker/test/api/start-attempt.test.ts @@ -29,11 +29,10 @@ test('loadAttempt should return an execution plan', async (t) => { }); const plan = await loadAttempt(channel); - t.deepEqual(plan, { + t.like(plan, { id: 'attempt-1', jobs: [ { - id: 'trigger', configuration: 'a', expression: 'fn(a => a)', adaptor: '@openfn/language-common@1.0.0', diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index 69d104f9e..f85f0a83c 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -340,14 +340,10 @@ test.serial( server.addDataclip('d', dataclips['d']); const result = { answer: 42 }; - server - .waitForResult(attempt1.id) - .then(({ attemptId, dataclip, logs }) => { - t.is(attemptId, attempt1.id); - t.deepEqual(result, dataclip); - t.deepEqual(logs, []); - done(); - }); + server.waitForResult(attempt1.id).then(({ attemptId, dataclip }) => { + t.deepEqual(result, dataclip); + done(); + }); const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); channel.push(ATTEMPT_COMPLETE, { dataclip: result }); From fd9c3746094904d3117ac789b81b939d665241a7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 2 Oct 2023 13:28:28 +0100 Subject: [PATCH 107/232] worker: refactor initial state handling --- packages/ws-worker/src/api/execute.ts | 14 ++-- packages/ws-worker/src/events.ts | 10 +-- .../src/mock/lightning/api-sockets.ts | 2 +- .../ws-worker/src/mock/lightning/server.ts | 5 +- packages/ws-worker/src/mock/runtime-engine.ts | 46 ++++++------- packages/ws-worker/src/types.d.ts | 8 +-- .../ws-worker/src/util/convert-attempt.ts | 11 ++++ packages/ws-worker/test/api/execute.test.ts | 17 ++--- .../ws-worker/test/api/start-attempt.test.ts | 1 + packages/ws-worker/test/integration.test.ts | 24 +++++-- packages/ws-worker/test/mock/data.ts | 1 + ...ntime-engine.ts => runtime-engine.test.ts} | 43 +++++++++++++ .../test/util/convert-attempt.test.ts | 64 +++++++++++++------ 13 files changed, 169 insertions(+), 77 deletions(-) rename packages/ws-worker/test/mock/{runtime-engine.ts => runtime-engine.test.ts} (85%) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index e344394b6..945d200b7 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -9,6 +9,7 @@ import { ATTEMPT_START_PAYLOAD, GET_CREDENTIAL, GET_DATACLIP, + GET_DATACLIP_PAYLOAD, RUN_COMPLETE, RUN_COMPLETE_PAYLOAD, RUN_START, @@ -51,7 +52,7 @@ export function execute( logger: Logger, plan: ExecutionPlan ) { - return new Promise((resolve) => { + return new Promise(async (resolve) => { // TODO add proper logger (maybe channel, rtm and logger comprise a context object) // tracking state for this attempt const state: AttemptState = { @@ -92,10 +93,15 @@ export function execute( engine.listen(plan.id, listeners); const resolvers = { - state: (id: string) => loadState(channel, id), credential: (id: string) => loadCredential(channel, id), + // dataclip: (id: string) => loadState(channel, id), + // TODO not supported right now }; + if (typeof plan.initialState === 'string') { + plan.initialState = await loadState(channel, plan.initialState); + } + engine.execute(plan, resolvers); }); } @@ -163,12 +169,12 @@ export function onJobLog({ channel, state }: Context, event: JSONLog) { export async function loadState(channel: Channel, stateId: string) { const result = await getWithReply(channel, GET_DATACLIP, { - dataclip_id: stateId, + id: stateId, }); const str = enc.decode(new Uint8Array(result)); return JSON.parse(str); } export async function loadCredential(channel: Channel, credentialId: string) { - return getWithReply(channel, GET_CREDENTIAL, { credential_id: credentialId }); + return getWithReply(channel, GET_CREDENTIAL, { id: credentialId }); } diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 51407d7cc..7bb90057d 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -1,4 +1,5 @@ import { JSONLog } from '@openfn/logger'; +import { Attempt } from './types'; // track socket event names as constants to keep refactoring easier @@ -11,17 +12,8 @@ export type CLAIM_ATTEMPT = { id: string; token: string }; export const GET_ATTEMPT = 'fetch:attempt'; export type GET_ATTEMPT_PAYLOAD = void; // no payload -// This is basically the attempt, which needs defining properly export type GET_ATTEMPT_REPLY = Attempt; -// TODO -type Attempt = { - id: string; - workflow: {}; - options: {}; - dataclip: string; -}; - export const GET_CREDENTIAL = 'fetch:credential'; export type GET_CREDENTIAL_PAYLOAD = { id: string }; // credential in-line, no wrapper, arbitrary data diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index 9401e0d2e..5a9c8f09d 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -175,7 +175,7 @@ const createSocketAPI = ( ) { const { ref, topic } = evt; const attemptId = extractAttemptId(topic); - const response = state.attempts[attemptId]; /// TODO this is badly typed + const response = state.attempts[attemptId]; ws.reply({ ref, diff --git a/packages/ws-worker/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts index a4986b1d2..08e14215d 100644 --- a/packages/ws-worker/src/mock/lightning/server.ts +++ b/packages/ws-worker/src/mock/lightning/server.ts @@ -11,6 +11,7 @@ import createLogger, { import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; +import { Attempt } from '../../types'; export const API_PREFIX = '/api/1'; @@ -25,8 +26,8 @@ export type ServerState = { // list of credentials by id credentials: Record; - // list of events by id - attempts: Record; + // list of attempts by id + attempts: Record; // list of dataclips by id dataclips: Record; diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 5d599893c..8425d88e1 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { ExecutionPlan } from '@openfn/runtime'; +import type { ExecutionPlan, JobNode } from '@openfn/runtime'; import type { State, Credential } from '../types'; import mockResolvers from './resolvers'; @@ -51,18 +51,11 @@ export type WorkflowCompleteEvent = { error?: any; }; -// TODO log event optionally has a job id - let jobId = 0; const getNewJobId = () => `${++jobId}`; let autoServerId = 0; -// Before we execute each job (expression), we have to build a state object -// This means squashing together the input state and the credential -// The credential of course is the hard bit -const assembleState = () => {}; - function createMock(serverId?: string) { const activeWorkflows = {} as Record; const bus = new EventEmitter(); @@ -93,7 +86,7 @@ function createMock(serverId?: string) { const executeJob = async ( workflowId: string, - job: JobPlan, + job: JobNode, initialState = {}, resolvers: LazyResolvers = mockResolvers ) => { @@ -105,11 +98,6 @@ function createMock(serverId?: string) { // Maybe later we use it to assemble state await resolvers.credential(configuration); } - if (typeof state === 'string') { - // TODO right now we lazy load any state object - // but maybe we need to just do initial state? - await resolvers.state(state); - } // Does a job reallly need its own id...? Maybe. const runId = getNewJobId(); @@ -123,10 +111,11 @@ function createMock(serverId?: string) { message: ['Running job ' + jobId], level: 'info', }); - let state = initialState; + let nextState = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { - state = JSON.parse(expression); + // @ts-ignore + nextState = JSON.parse(expression); // What does this look like? Should be a logger object dispatch('log', { workflowId, @@ -134,32 +123,43 @@ function createMock(serverId?: string) { message: ['Parsing expression as JSON state'], level: 'info', }); - dispatch('log', { workflowId, jobId, message: [state], level: 'info' }); + dispatch('log', { + workflowId, + jobId, + message: [nextState], + level: 'info', + }); } catch (e) { // Do nothing, it's fine + nextState = initialState; } - dispatch('job-complete', { workflowId, jobId, state, runId }); + dispatch('job-complete', { workflowId, jobId, state: nextState, runId }); - return state; + return nextState; }; // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity - const execute = ( + const execute = async ( xplan: ExecutionPlan, resolvers: LazyResolvers = mockResolvers ) => { - const { id, jobs } = xplan; + const { id, jobs, initialState } = xplan; const workflowId = id; activeWorkflows[id!] = true; + + // TODO do we want to load a dataclip from job.state here? + // This isn't supported right now + // We would need to use resolvers.dataclip if we wanted it + setTimeout(() => { dispatch('workflow-start', { workflowId }); setTimeout(async () => { - let state = {}; + let state = initialState || {}; // Trivial job reducer in our mock for (const job of jobs) { - state = await executeJob(id, job, state, resolvers); + state = await executeJob(id!, job, state, resolvers); } setTimeout(() => { delete activeWorkflows[id!]; diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index 7f273a897..ca0659872 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -35,18 +35,16 @@ export interface Edge { } // An attempt object returned by Lightning -// We may later drop this abstraction and just accept an excecution plan directly export type Attempt = { id: string; + dataclip_id: string; + starting_node_id: string; triggers: Node[]; jobs: Node[]; edges: Edge[]; - // these will probably be included by lightning but we don't care here - projectId?: string; - status?: string; - worker?: string; + options?: Record; // TODO type the expected options }; export type CancelablePromise = Promise & { diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index fb93f27f0..46d26e627 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -7,6 +7,17 @@ export default (attempt: Attempt): ExecutionPlan => { id: attempt.id, }; + if (attempt.dataclip_id) { + // This is tricky - we're assining a string to the XPlan + // which is fine becuase it'll be handled later + // I guess we need a new type for now? Like a lazy XPlan + // @ts-ignore + plan.initialState = attempt.dataclip_id; + } + if (attempt.starting_node_id) { + plan.start = attempt.starting_node_id; + } + const nodes: Record = {}; const edges = attempt.edges ?? []; diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index c20cf3278..b1f0c45d3 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -24,6 +24,7 @@ import { import createMockRTE from '../../src/mock/runtime-engine'; import { mockChannel } from '../../src/mock/sockets'; import { stringify } from '../../src/util'; +import { ExecutionPlan } from '@openfn/runtime'; const enc = new TextEncoder(); @@ -240,8 +241,8 @@ test('workflowComplete should call onComplete with state', async (t) => { // TODO what if an error? test('loadState should fetch a dataclip', async (t) => { const channel = mockChannel({ - [GET_DATACLIP]: ({ dataclip_id }) => { - t.is(dataclip_id, 'xyz'); + [GET_DATACLIP]: ({ id }) => { + t.is(id, 'xyz'); return toArrayBuffer({ data: {} }); }, }); @@ -253,8 +254,8 @@ test('loadState should fetch a dataclip', async (t) => { // TODO what if an error? test('loadCredential should fetch a credential', async (t) => { const channel = mockChannel({ - [GET_CREDENTIAL]: ({ credential_id }) => { - t.is(credential_id, 'jfk'); + [GET_CREDENTIAL]: ({ id }) => { + t.is(id, 'jfk'); return { apiKey: 'abc' }; }, }); @@ -311,7 +312,6 @@ test('execute should lazy-load a credential', async (t) => { t.true(didCallCredentials); }); -// TODO this is more of an engine test really, but worth having I suppose test('execute should lazy-load initial state', async (t) => { const logger = createMockLogger(); let didCallState = false; @@ -325,17 +325,18 @@ test('execute should lazy-load initial state', async (t) => { }); const engine = createMockRTE('rte'); - const plan = { + const plan: Partial = { id: 'a', + // @ts-ignore + initialState: 'abc', jobs: [ { - state: 'abc', expression: JSON.stringify({ done: true }), }, ], }; - await execute(channel, engine, logger, plan); + await execute(channel, engine, logger, plan as ExecutionPlan); t.true(didCallState); }); diff --git a/packages/ws-worker/test/api/start-attempt.test.ts b/packages/ws-worker/test/api/start-attempt.test.ts index 914ccfbaa..e5d92197c 100644 --- a/packages/ws-worker/test/api/start-attempt.test.ts +++ b/packages/ws-worker/test/api/start-attempt.test.ts @@ -33,6 +33,7 @@ test('loadAttempt should return an execution plan', async (t) => { id: 'attempt-1', jobs: [ { + id: 'job-1', configuration: 'a', expression: 'fn(a => a)', adaptor: '@openfn/language-common@1.0.0', diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 2a5e135ab..2e9d553bc 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -39,6 +39,10 @@ const getAttempt = (ext = {}, jobs?: any) => ({ ...ext, }); +// these are really just tests of the mock architecture, but worth having +test.todo('should run an attempt which returns from json'); +test.todo('should run an attempt which returns intial state'); + // A basic high level integration test to ensure the whole loop works // This checks the events received by the lightning websocket test.serial( @@ -102,7 +106,7 @@ test.serial( didCallEvent = true; }); - lng.onSocketEvent(e.GET_ATTEMPT, attempt.id, (evt) => { + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { t.true(didCallEvent); done(); }); @@ -131,7 +135,7 @@ test.serial( didCallEvent = true; }); - lng.onSocketEvent(e.GET_CREDENTIAL, attempt.id, (evt) => { + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { t.true(didCallEvent); done(); }); @@ -141,21 +145,27 @@ test.serial( } ); -test.serial.skip( +test.serial( `events: lightning should receive a ${e.GET_DATACLIP} event`, (t) => { return new Promise((done) => { + lng.addDataclip('abc', { result: true }); + const attempt = getAttempt({ - dataclip_id: 'abc', // TODO this isn't implemented yet + dataclip_id: 'abc', }); let didCallEvent = false; lng.onSocketEvent(e.GET_DATACLIP, attempt.id, ({ payload }) => { - // again there's no way to check the right credential was returned + // payload is the incoming/request payload - this tells us which dataclip + // the worker is asking for + // Note that it doesn't tell us much about what is returned + // (and we can't tell from this event either) + t.is(payload.id, 'abc'); didCallEvent = true; }); - lng.onSocketEvent(e.GET_DATACLIP, attempt.id, (evt) => { + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, () => { t.true(didCallEvent); done(); }); @@ -178,7 +188,7 @@ test.serial.skip( didCallEvent = true; }); - lng.onSocketEvent(e.RUN_START, attempt.id, (evt) => { + lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { t.true(didCallEvent); done(); }); diff --git a/packages/ws-worker/test/mock/data.ts b/packages/ws-worker/test/mock/data.ts index 121107f31..e6e773be6 100644 --- a/packages/ws-worker/test/mock/data.ts +++ b/packages/ws-worker/test/mock/data.ts @@ -22,6 +22,7 @@ export const attempts = { edges: [], jobs: [ { + id: 'job-1', adaptor: '@openfn/language-common@1.0.0', body: 'fn(a => a)', credential: 'a', diff --git a/packages/ws-worker/test/mock/runtime-engine.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts similarity index 85% rename from packages/ws-worker/test/mock/runtime-engine.ts rename to packages/ws-worker/test/mock/runtime-engine.test.ts index 6ba8a9d78..b10769437 100644 --- a/packages/ws-worker/test/mock/runtime-engine.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -100,6 +100,47 @@ test('mock should evaluate expressions as JSON', async (t) => { t.deepEqual(evt.state, { x: 10 }); }); +test('mock should return initial state as result state', async (t) => { + const engine = create(); + + const wf = { + initialState: { y: 22 }, + jobs: [ + { + adaptor: 'common@1.0.0', + }, + ], + }; + engine.execute(wf); + + const evt = await waitForEvent( + engine, + 'workflow-complete' + ); + t.deepEqual(evt.state, { y: 22 }); +}); + +test('mock prefers JSON state to initial state', async (t) => { + const engine = create(); + + const wf = { + initialState: { y: 22 }, + jobs: [ + { + adaptor: 'common@1.0.0', + expression: '{ "z": 33 }', + }, + ], + }; + engine.execute(wf); + + const evt = await waitForEvent( + engine, + 'workflow-complete' + ); + t.deepEqual(evt.state, { z: 33 }); +}); + test('mock should dispatch log events when evaluating JSON', async (t) => { const engine = create(); @@ -189,3 +230,5 @@ test('only listen to events for the correct workflow', async (t) => { await waitForEvent(engine, 'workflow-complete'); t.pass(); }); + +// test('inital dataclip') diff --git a/packages/ws-worker/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts index 24d6f9d65..7518b2c32 100644 --- a/packages/ws-worker/test/util/convert-attempt.test.ts +++ b/packages/ws-worker/test/util/convert-attempt.test.ts @@ -37,13 +37,13 @@ const createJob = (props = {}) => ({ }); test('convert a single job', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [createNode()], triggers: [], edges: [], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -54,13 +54,13 @@ test('convert a single job', (t) => { // Note idk how lightningg will handle state/defaults on a job // but this is what we'll do right now test('convert a single job with data', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [createNode({ state: { data: { x: 22 } } })], triggers: [], edges: [], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -80,14 +80,42 @@ test('Accept a partial attempt object', (t) => { }); }); +test('handle dataclip_id', (t) => { + const attempt: Partial = { + id: 'w', + dataclip_id: 'xyz', + }; + const result = convertAttempt(attempt as Attempt); + + t.deepEqual(result, { + id: 'w', + initialState: 'xyz', + jobs: [], + }); +}); + +test('handle starting_node_id', (t) => { + const attempt: Partial = { + id: 'w', + starting_node_id: 'j1', + }; + const result = convertAttempt(attempt as Attempt); + + t.deepEqual(result, { + id: 'w', + start: 'j1', + jobs: [], + }); +}); + test('convert a single trigger', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', triggers: [createTrigger()], jobs: [], edges: [], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -101,13 +129,13 @@ test('convert a single trigger', (t) => { // This exhibits current behaviour. This should never happen though test('ignore a single edge', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [], triggers: [], edges: [createEdge('a', 'b')], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -116,7 +144,7 @@ test('ignore a single edge', (t) => { }); test('convert a single trigger with an edge', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', triggers: [createTrigger()], jobs: [createNode()], @@ -128,7 +156,7 @@ test('convert a single trigger with an edge', (t) => { }, ], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -145,7 +173,7 @@ test('convert a single trigger with an edge', (t) => { }); test('convert a single trigger with two edges', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', triggers: [createTrigger()], jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], @@ -162,7 +190,7 @@ test('convert a single trigger with two edges', (t) => { }, ], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -181,13 +209,13 @@ test('convert a single trigger with two edges', (t) => { }); test('convert two linked jobs', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], triggers: [], edges: [createEdge('a', 'b')], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -197,7 +225,7 @@ test('convert two linked jobs', (t) => { // This isn't supported by the runtime, but it'll survive the conversion test('convert a job with two upstream jobs', (t) => { - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [ createNode({ id: 'a' }), @@ -207,7 +235,7 @@ test('convert a job with two upstream jobs', (t) => { triggers: [], edges: [createEdge('a', 'x'), createEdge('b', 'x')], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', @@ -221,13 +249,13 @@ test('convert a job with two upstream jobs', (t) => { test('convert two linked jobs with an edge condition', (t) => { const condition = 'state.age > 10'; - const attempt: Attempt = { + const attempt: Partial = { id: 'w', jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], triggers: [], edges: [createEdge('a', 'b', { condition })], }; - const result = convertAttempt(attempt); + const result = convertAttempt(attempt as Attempt); t.deepEqual(result, { id: 'w', From 703a7d98ac8c25ad5619b8e8be9f08e78b1c3c9c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 2 Oct 2023 15:11:58 +0100 Subject: [PATCH 108/232] runtime: accept initial state on execution plan --- packages/runtime/src/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 1faa8651d..8d8545994 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -42,8 +42,9 @@ export declare interface Operation | State> { export type ExecutionPlan = { id?: string; // UUID for this plan - start?: JobNodeID; jobs: JobNode[]; + start?: JobNodeID; + initialState?: State; // TODO adding initial state to the plan changes how the runtime expects to receive initial state }; export type JobNode = { From bfc74e9d8970769565c9fef52b922492ee0e7882 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 2 Oct 2023 16:50:39 +0100 Subject: [PATCH 109/232] worker: wired up end-to-end tests --- packages/ws-worker/src/api/execute.ts | 31 +++++++--- packages/ws-worker/src/events.ts | 6 +- .../ws-worker/src/mock/lightning/api-dev.ts | 4 +- .../src/mock/lightning/api-sockets.ts | 57 +++++++++++++++++-- packages/ws-worker/src/mock/runtime-engine.ts | 18 +++--- packages/ws-worker/test/api/execute.test.ts | 48 +++++++++++++--- packages/ws-worker/test/integration.test.ts | 52 +++++++++++++++-- .../ws-worker/test/mock/lightning.test.ts | 12 ++-- .../test/mock/runtime-engine.test.ts | 19 ++----- 9 files changed, 190 insertions(+), 57 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 945d200b7..5d49a9579 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -31,10 +31,10 @@ export type AttemptState = { activeRun?: string; activeJob?: string; plan: ExecutionPlan; - // final state/dataclip - result?: any; + dataclips: Record; - // TODO status? + // final dataclip id + result?: string; }; type Context = { @@ -121,13 +121,28 @@ export function onJobStart({ channel, state }: Context, event: any) { } export function onJobComplete({ channel, state }: Context, event: any) { + const dataclipId = crypto.randomUUID(); + channel.push(RUN_COMPLETE, { run_id: state.activeRun!, job_id: state.activeJob!, - // TODO generate a dataclip id + output_dataclip_id: dataclipId, output_dataclip: stringify(event.state), }); + if (!state.dataclips) { + state.dataclips = {}; + } + state.dataclips[dataclipId] = event.state; + + // TODO right now, the last job to run will be the result for the attempt + // this may not stand up in the future + // I'd feel happer if the runtime could judge what the final result is + // (taking into account branches and stuff) + // The problem is that the runtime will return the object, not an id, + // so we have a bit of a mapping problem + state.result = dataclipId; + delete state.activeRun; delete state.activeJob; } @@ -141,16 +156,16 @@ export function onWorkflowStart( export function onWorkflowComplete( { state, channel, onComplete }: Context, - event: WorkflowCompleteEvent + _event: WorkflowCompleteEvent ) { - state.result = event.state; + const result = state.dataclips[state.result!]; channel .push(ATTEMPT_COMPLETE, { - dataclip: stringify(event.state), // TODO this should just be dataclip id + final_dataclip_id: state.result!, }) .receive('ok', () => { - onComplete(state.result); + onComplete(result); }); } diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 7bb90057d..f3af5cd61 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -28,7 +28,10 @@ export type ATTEMPT_START_PAYLOAD = void; // no payload export type ATTEMPT_START_REPLY = void; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats -export type ATTEMPT_COMPLETE_PAYLOAD = { dataclip: any; stats?: any }; // TODO dataclip -> result? output_dataclip? +export type ATTEMPT_COMPLETE_PAYLOAD = { + final_dataclip_id: string; + stats?: any; +}; // TODO dataclip -> result? output_dataclip? export type ATTEMPT_COMPLETE_REPLY = undefined; export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time @@ -57,5 +60,6 @@ export type RUN_COMPLETE_PAYLOAD = { job_id: string; run_id: string; output_dataclip?: string; + output_dataclip_id?: string; }; export type RUN_COMPLETE_REPLY = void; diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts index 78627e5bb..43a327b83 100644 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ b/packages/ws-worker/src/mock/lightning/api-dev.ts @@ -65,8 +65,8 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { }) => { if (evt.attemptId === attemptId) { state.events.removeListener(ATTEMPT_COMPLETE, handler); - - resolve(evt.payload); + const result = state.dataclips[evt.payload.final_dataclip_id]; + resolve(result); } }; state.events.addListener(ATTEMPT_COMPLETE, handler); diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index 5a9c8f09d..3d483ded4 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -28,6 +28,11 @@ import { GET_DATACLIP, GET_DATACLIP_PAYLOAD, GET_DATACLIP_REPLY, + RUN_COMPLETE, + RUN_COMPLETE_PAYLOAD, + RUN_START, + RUN_START_PAYLOAD, + RUN_START_REPLY, } from '../../events'; import type { Server } from 'http'; @@ -114,15 +119,13 @@ const createSocketAPI = ( [GET_ATTEMPT]: wrap(getAttempt), [GET_CREDENTIAL]: wrap(getCredential), [GET_DATACLIP]: wrap(getDataclip), + [RUN_START]: wrap(handleRunStart), [ATTEMPT_LOG]: wrap(handleLog), + [RUN_COMPLETE]: wrap(handleRunComplete), [ATTEMPT_COMPLETE]: wrap((...args) => { handleAttemptComplete(...args); unsubscribe(); }), - - // TODO - // [RUN_START] - // [RUN_COMPLETE] }); }; @@ -260,6 +263,52 @@ const createSocketAPI = ( state.pending[attemptId].status = 'complete'; state.results[attemptId] = dataclip; + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + // TODO final dataclip id + // this is kind of awkward to work out + // we gotta sha every dataclip, find a match, then return + }, + }); + } + + function handleRunStart( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, topic } = evt; + if (!state.dataclips) { + state.dataclips = {}; + } + ws.reply({ + ref, + topic, + payload: { + status: 'ok', + }, + }); + } + + function handleRunComplete( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, topic, payload } = evt; + const { output_dataclip_id, output_dataclip } = evt.payload; + + if (output_dataclip_id) { + if (!state.dataclips) { + state.dataclips = {}; + } + state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip); + } + + // be polite and acknowledge the event ws.reply({ ref, topic, diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 8425d88e1..af9b29aa6 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,3 +1,4 @@ +import crypto from 'node:crypto'; import { EventEmitter } from 'node:events'; import type { ExecutionPlan, JobNode } from '@openfn/runtime'; @@ -47,13 +48,9 @@ export type WorkflowStartEvent = { export type WorkflowCompleteEvent = { workflowId: string; - state?: object; error?: any; }; -let jobId = 0; -const getNewJobId = () => `${++jobId}`; - let autoServerId = 0; function createMock(serverId?: string) { @@ -90,8 +87,10 @@ function createMock(serverId?: string) { initialState = {}, resolvers: LazyResolvers = mockResolvers ) => { - // TODO maybe lazy load the job from an id - const { id, expression, configuration, state } = job; + const { id, expression, configuration } = job; + + const runId = crypto.randomUUID(); + const jobId = id; if (typeof configuration === 'string') { // Fetch the credential but do nothing with it @@ -99,9 +98,6 @@ function createMock(serverId?: string) { await resolvers.credential(configuration); } - // Does a job reallly need its own id...? Maybe. - const runId = getNewJobId(); - // Get the job details from lightning // start instantly and emit as it goes dispatch('job-start', { workflowId, jobId, runId }); @@ -149,7 +145,7 @@ function createMock(serverId?: string) { const workflowId = id; activeWorkflows[id!] = true; - // TODO do we want to load a dataclip from job.state here? + // TODO do we want to load a globals dataclip from job.state here? // This isn't supported right now // We would need to use resolvers.dataclip if we wanted it @@ -163,7 +159,7 @@ function createMock(serverId?: string) { } setTimeout(() => { delete activeWorkflows[id!]; - dispatch('workflow-complete', { workflowId, state }); + dispatch('workflow-complete', { workflowId }); // TODO on workflow complete we should maybe tidy the listeners? // Doesn't really matter in the mock though }, 1); diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index b1f0c45d3..2085d7755 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -87,6 +87,28 @@ test('jobComplete should clear the run id and active job on state', async (t) => t.falsy(state.activeRun); }); +test('jobComplete should save the dataclip to state', async (t) => { + const plan = { id: 'attempt-1' }; + const jobId = 'job-1'; + + const state = { + plan, + activeJob: jobId, + activeRun: 'b', + dataclips: {}, + result: undefined, + } as AttemptState; + + const channel = mockChannel({}); + + const event = { state: { x: 10 } }; + onJobComplete({ channel, state }, event); + + t.is(Object.keys(state.dataclips).length, 1); + const [dataclip] = Object.values(state.dataclips); + t.deepEqual(dataclip, event.state); +}); + test('jobComplete should send a run:complete event', async (t) => { return new Promise((done) => { const plan = { id: 'attempt-1' }; @@ -103,6 +125,7 @@ test('jobComplete should send a run:complete event', async (t) => { [RUN_COMPLETE]: (evt) => { t.is(evt.job_id, jobId); t.truthy(evt.run_id); + t.truthy(evt.output_dataclip_id); t.is(evt.output_dataclip, JSON.stringify(result)); done(); @@ -195,32 +218,41 @@ test('workflowStart should send an empty attempt:start event', async (t) => { test('workflowComplete should send an attempt:complete event', async (t) => { return new Promise((done) => { - const state = {} as AttemptState; - const result = { answer: 42 }; + const state = { + dataclips: { + x: result, + }, + result: 'x', + }; + const channel = mockChannel({ [ATTEMPT_COMPLETE]: (evt) => { - t.deepEqual(evt.dataclip, JSON.stringify(result)); - t.deepEqual(state.result, result); + t.deepEqual(evt.final_dataclip_id, 'x'); done(); }, }); - const event = { state: result }; + const event = {}; const context = { channel, state, onComplete: () => {} }; onWorkflowComplete(context, event); }); }); -test('workflowComplete should call onComplete with state', async (t) => { +test('workflowComplete should call onComplete with final dataclip', async (t) => { return new Promise((done) => { - const state = {} as AttemptState; - const result = { answer: 42 }; + const state = { + dataclips: { + x: result, + }, + result: 'x', + }; + const channel = mockChannel(); const context = { diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 2e9d553bc..b5c1528e8 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -40,8 +40,52 @@ const getAttempt = (ext = {}, jobs?: any) => ({ }); // these are really just tests of the mock architecture, but worth having -test.todo('should run an attempt which returns from json'); -test.todo('should run an attempt which returns intial state'); +test.serial( + 'should run an attempt through the mock runtime which returns an expression as JSON', + async (t) => { + return new Promise((done) => { + const attempt = { + id: 'attempt-1', + jobs: [ + { + body: JSON.stringify({ count: 122 }), + }, + ], + }; + + lng.waitForResult(attempt.id).then((result) => { + t.deepEqual(result, { count: 122 }); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); +test.serial('should run an attempt which returns intial state', async (t) => { + return new Promise((done) => { + lng.addDataclip('x', { + route: 66, + }); + + const attempt = { + id: 'attempt-1', + dataclip_id: 'x', + jobs: [ + { + body: 'whatever', + }, + ], + }; + + lng.waitForResult(attempt.id).then((result) => { + t.deepEqual(result, { route: 66 }); + done(); + }); + + lng.enqueueAttempt(attempt); + }); +}); // A basic high level integration test to ensure the whole loop works // This checks the events received by the lightning websocket @@ -51,8 +95,8 @@ test.serial( return new Promise((done) => { const attempt = getAttempt(); lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, (evt) => { - // TODO we should validate the result event here, but it's not quite decided - // I think it should be { attempt_id, dataclip_id } + const { final_dataclip_id } = evt.payload; + t.assert(typeof final_dataclip_id === 'string'); t.pass('attempt complete event received'); done(); }); diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index f85f0a83c..facc075f5 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -6,6 +6,7 @@ import phx from 'phoenix-channels'; import { attempts, credentials, dataclips } from './data'; import { ATTEMPT_COMPLETE, + ATTEMPT_COMPLETE_PAYLOAD, ATTEMPT_LOG, CLAIM, GET_ATTEMPT, @@ -336,17 +337,20 @@ test.serial( 'waitForResult should return logs and dataclip when an attempt is completed', async (t) => { return new Promise(async (done) => { - server.startAttempt(attempt1.id); - server.addDataclip('d', dataclips['d']); const result = { answer: 42 }; - server.waitForResult(attempt1.id).then(({ attemptId, dataclip }) => { + server.startAttempt(attempt1.id); + server.addDataclip('result', result); + + server.waitForResult(attempt1.id).then((dataclip) => { t.deepEqual(result, dataclip); done(); }); const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_COMPLETE, { dataclip: result }); + channel.push(ATTEMPT_COMPLETE, { + final_dataclip_id: 'result', + } as ATTEMPT_COMPLETE_PAYLOAD); }); } ); diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index b10769437..a19aa037b 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -63,9 +63,7 @@ test('Dispatch complete events when a workflow completes', async (t) => { 'workflow-complete' ); - t.truthy(evt); - t.is(evt.workflowId, 'w1'); - t.truthy(evt.state); + t.deepEqual(evt, { workflowId: 'w1' }); }); test('Dispatch start events for a job', async (t) => { @@ -93,10 +91,7 @@ test('mock should evaluate expressions as JSON', async (t) => { const engine = create(); engine.execute(sampleWorkflow); - const evt = await waitForEvent( - engine, - 'workflow-complete' - ); + const evt = await waitForEvent(engine, 'job-complete'); t.deepEqual(evt.state, { x: 10 }); }); @@ -113,10 +108,7 @@ test('mock should return initial state as result state', async (t) => { }; engine.execute(wf); - const evt = await waitForEvent( - engine, - 'workflow-complete' - ); + const evt = await waitForEvent(engine, 'job-complete'); t.deepEqual(evt.state, { y: 22 }); }); @@ -134,10 +126,7 @@ test('mock prefers JSON state to initial state', async (t) => { }; engine.execute(wf); - const evt = await waitForEvent( - engine, - 'workflow-complete' - ); + const evt = await waitForEvent(engine, 'job-complete'); t.deepEqual(evt.state, { z: 33 }); }); From fa2b51d6b61bb842f7ed70b8c113f4209f8fbe88 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 3 Oct 2023 11:42:19 +0100 Subject: [PATCH 110/232] worker: OPENFN_WORKER_SECRET -> WORKER_SECRET --- packages/ws-worker/README.md | 2 +- packages/ws-worker/src/start.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index 66ea922db..ee9dc8d6e 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -17,7 +17,7 @@ The mock services allow lightweight and controlled testing of the interfaces bet There are several components you may want to run to get started. -If you're running a local lightning server, remember to set OPENFN_WORKER_SECRET. +If you're running a local lightning server, remember to set WORKER_SECRET. ### WS Server diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 07dffbf8f..ed6f8553b 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -36,10 +36,10 @@ const args = yargs(hideBin(process.argv)) }) .option('secret', { alias: 's', - description: 'Worker secret (comes from OPENFN_WORKER_SECRET by default)', + description: 'Worker secret (comes from WORKER_SECRET by default)', }) .option('log', { - description: 'Worker secret (comes from OPENFN_WORKER_SECRET by default)', + description: 'Worker secret (comes from WORKER_SECRET by default)', default: 'info', type: 'string', }) @@ -50,13 +50,13 @@ const logger = createLogger('SRV', { level: args.log }); if (args.lightning === 'mock') { args.lightning = 'ws://localhost:8888/api'; } else if (!args.secret) { - const { OPENFN_WORKER_SECRET } = process.env; - if (!OPENFN_WORKER_SECRET) { - logger.error('OPENFN_WORKER_SECRET is not set'); + const { WORKER_SECRET } = process.env; + if (!WORKER_SECRET) { + logger.error('WORKER_SECRET is not set'); process.exit(1); } - args.secret = OPENFN_WORKER_SECRET; + args.secret = WORKER_SECRET; } // TODO the engine needs to take callbacks to load credential, and load state From 6b8df8483b61cf026aa78acc8af78f3eeac338f0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 3 Oct 2023 11:58:09 +0100 Subject: [PATCH 111/232] worker: endpoint at /worker --- packages/ws-worker/src/mock/lightning/index.ts | 2 -- packages/ws-worker/src/mock/lightning/server.ts | 4 +--- packages/ws-worker/src/start.ts | 4 ++-- packages/ws-worker/test/integration.test.ts | 2 +- packages/ws-worker/test/mock/lightning.test.ts | 4 ++-- packages/ws-worker/test/socket-client.js | 2 +- 6 files changed, 7 insertions(+), 11 deletions(-) diff --git a/packages/ws-worker/src/mock/lightning/index.ts b/packages/ws-worker/src/mock/lightning/index.ts index b205885b6..5f3b83d4f 100644 --- a/packages/ws-worker/src/mock/lightning/index.ts +++ b/packages/ws-worker/src/mock/lightning/index.ts @@ -1,4 +1,2 @@ import createLightningServer from './server'; export default createLightningServer; - -export { API_PREFIX } from './server'; diff --git a/packages/ws-worker/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts index 08e14215d..3d1ac66d7 100644 --- a/packages/ws-worker/src/mock/lightning/server.ts +++ b/packages/ws-worker/src/mock/lightning/server.ts @@ -13,8 +13,6 @@ import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; -export const API_PREFIX = '/api/1'; - export type AttemptState = { status: 'queued' | 'started' | 'complete'; logs: JSONLog[]; @@ -84,7 +82,7 @@ const createLightningServer = (options: LightningOptions = {}) => { // Setup the websocket API const api = createWebSocketAPI( state, - '/api', + '/worker', // TODO I should option drive this server, options.logger && logger ); diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index ed6f8553b..63a2010d6 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -28,7 +28,7 @@ const args = yargs(hideBin(process.argv)) .option('lightning', { alias: 'l', description: - 'Base url to Lightning websocket endpoint, eg, ws://locahost:4000/api. Set to "mock" to use the default mock server', + 'Base url to Lightning websocket endpoint, eg, ws://locahost:4000/worker. Set to "mock" to use the default mock server', }) .option('repo-dir', { alias: 'd', @@ -48,7 +48,7 @@ const args = yargs(hideBin(process.argv)) const logger = createLogger('SRV', { level: args.log }); if (args.lightning === 'mock') { - args.lightning = 'ws://localhost:8888/api'; + args.lightning = 'ws://localhost:8888/worker'; } else if (!args.secret) { const { WORKER_SECRET } = process.env; if (!WORKER_SECRET) { diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index b5c1528e8..045afef7b 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -13,7 +13,7 @@ let engine; const urls = { engine: 'http://localhost:4567', - lng: 'ws://localhost:7654/api', + lng: 'ws://localhost:7654/worker', }; test.before(() => { diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index facc075f5..25e994438 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -16,7 +16,7 @@ import { import type { Attempt } from '../../src/types'; import { JSONLog } from '@openfn/logger'; -const endpoint = 'ws://localhost:7777/api'; +const endpoint = 'ws://localhost:7777/worker'; const enc = new TextDecoder('utf-8'); @@ -113,7 +113,7 @@ test.serial('should setup an attempt at /POST /attempt', async (t) => { t.is(c.user, 'john'); }); -test.serial('provide a phoenix websocket at /api', (t) => { +test.serial('provide a phoenix websocket at /worker', (t) => { // client should be connected before this test runs t.is(client.connectionState(), 'open'); }); diff --git a/packages/ws-worker/test/socket-client.js b/packages/ws-worker/test/socket-client.js index fd3f83dbf..83ab39e52 100644 --- a/packages/ws-worker/test/socket-client.js +++ b/packages/ws-worker/test/socket-client.js @@ -2,7 +2,7 @@ // run from the commandline ie `node test/socket-client.js` import phx from 'phoenix-channels'; -const endpoint = 'ws://localhost:8888/api'; +const endpoint = 'ws://localhost:8888/worker'; console.log('connecting to socket at ', endpoint); const socket = new phx.Socket(endpoint); From a950f40fb9d134c37e6c82a0d971389e4e272005 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 3 Oct 2023 15:43:57 +0100 Subject: [PATCH 112/232] worker: messy commit... 1) fixed some events to speak to lightniong 2) added a debug api to disable auto-fetch form workers --- packages/ws-worker/src/api/claim.ts | 46 +++++++++++++++++++ packages/ws-worker/src/api/connect.ts | 2 +- packages/ws-worker/src/api/start-attempt.ts | 2 +- packages/ws-worker/src/api/workloop.ts | 34 +++----------- packages/ws-worker/src/events.ts | 5 +- .../src/mock/lightning/api-sockets.ts | 16 ++++--- packages/ws-worker/src/server.ts | 36 +++++++++++++-- packages/ws-worker/src/start.ts | 8 ++++ packages/ws-worker/test/api/workloop.test.ts | 11 +++-- .../ws-worker/test/mock/lightning.test.ts | 23 ++++------ packages/ws-worker/test/socket-client.js | 2 +- 11 files changed, 122 insertions(+), 63 deletions(-) create mode 100644 packages/ws-worker/src/api/claim.ts diff --git a/packages/ws-worker/src/api/claim.ts b/packages/ws-worker/src/api/claim.ts new file mode 100644 index 000000000..65a9fe84f --- /dev/null +++ b/packages/ws-worker/src/api/claim.ts @@ -0,0 +1,46 @@ +import { Logger, createMockLogger } from '@openfn/logger'; +import { CLAIM, CLAIM_ATTEMPT, CLAIM_PAYLOAD, CLAIM_REPLY } from '../events'; + +import type { Channel } from '../types'; + +const mockLogger = createMockLogger(); + +// TODO: this needs standalone unit tests now that it's bene moved +const claim = ( + channel: Channel, + execute: (attempt: CLAIM_ATTEMPT) => void, + logger: Logger = mockLogger +) => { + return new Promise((resolve, reject) => { + logger.debug('requesting attempt...'); + channel + .push(CLAIM, { demand: 1 }) + .receive('ok', ({ attempts }: CLAIM_REPLY) => { + logger.debug('pull ok', attempts); + // TODO what if we get here after we've been cancelled? + // the events have already been claimed... + + if (!attempts?.length) { + logger.debug('no attempts, backing off'); + // throw to backoff and try again + return reject(new Error('claim failed')); + } + + attempts.forEach((attempt) => { + logger.debug('starting attempt', attempt.id); + execute(attempt); + resolve(); + }); + }) + // TODO need implementations for both of these really + // What do we do if we fail to join the worker channel? + .receive('error', (r) => { + logger.debug('pull err'); + }) + .receive('timeout', (r) => { + logger.debug('pull timeout'); + }); + }); +}; + +export default claim; diff --git a/packages/ws-worker/src/api/connect.ts b/packages/ws-worker/src/api/connect.ts index f029f89eb..913126e56 100644 --- a/packages/ws-worker/src/api/connect.ts +++ b/packages/ws-worker/src/api/connect.ts @@ -27,7 +27,7 @@ export const connectToLightning = ( socket.onOpen(() => { // join the queue channel // TODO should this send the worker token? - const channel = socket.channel('attempts:queue'); + const channel = socket.channel('worker:queue'); channel .join() diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index a33747762..1f9e3b5e4 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -10,7 +10,7 @@ import type { Logger } from '@openfn/logger'; // TODO what happens if this channel join fails? // Lightning could vanish, channel could error on its side, or auth could be wrong -// We don't have a good feedback mechanism yet - attempts:queue is the only channel +// We don't have a good feedback mechanism yet - worker:queue is the only channel // we can feedback to // Maybe we need a general errors channel const joinAttemptChannel = ( diff --git a/packages/ws-worker/src/api/workloop.ts b/packages/ws-worker/src/api/workloop.ts index 3e1b74605..0c182824f 100644 --- a/packages/ws-worker/src/api/workloop.ts +++ b/packages/ws-worker/src/api/workloop.ts @@ -1,8 +1,10 @@ -import type { Logger } from '@openfn/logger'; -import { CLAIM, CLAIM_ATTEMPT, CLAIM_PAYLOAD, CLAIM_REPLY } from '../events'; +import { CLAIM_ATTEMPT } from '../events'; import tryWithBackoff, { Options } from '../util/try-with-backoff'; import type { CancelablePromise, Channel } from '../types'; +import type { Logger } from '@openfn/logger'; + +import claim from './claim'; // TODO this needs to return some kind of cancel function const startWorkloop = ( @@ -14,34 +16,10 @@ const startWorkloop = ( let promise: CancelablePromise; let cancelled = false; - const request = () => { - return new Promise((resolve, reject) => { - logger.debug('pull claim'); - channel - .push(CLAIM, { demand: 1 }) - .receive('ok', (attempts: CLAIM_REPLY) => { - // TODO what if we get here after we've been cancelled? - // the events have already been claimed... - - if (!attempts?.length) { - logger.debug('no attempts, backing off'); - // throw to backoff and try again - return reject(new Error('backoff')); - } - - attempts.forEach((attempt) => { - logger.debug('starting attempt', attempt.id); - execute(attempt); - resolve(); - }); - }); - }); - }; - const workLoop = () => { if (!cancelled) { - promise = tryWithBackoff(request, { - timeout: options.delay, + promise = tryWithBackoff(() => claim(channel, execute, logger), { + timeout: options.timeout, maxBackoff: options.maxBackoff, }); // TODO this needs more unit tests I think diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index f3af5cd61..9e94ce6b2 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -3,11 +3,12 @@ import { Attempt } from './types'; // track socket event names as constants to keep refactoring easier -export const CLAIM = 'attempt:claim'; +export const CLAIM = 'claim'; + // this is the lightning payload // TODO why are the types in all caps...? export type CLAIM_PAYLOAD = { demand?: number }; -export type CLAIM_REPLY = Array; +export type CLAIM_REPLY = { attempts: Array }; export type CLAIM_ATTEMPT = { id: string; token: string }; export const GET_ATTEMPT = 'fetch:attempt'; diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index 3d483ded4..daa0a2016 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -19,6 +19,7 @@ import { CLAIM, CLAIM_PAYLOAD, CLAIM_REPLY, + CLAIM_ATTEMPT, GET_ATTEMPT, GET_ATTEMPT_PAYLOAD, GET_ATTEMPT_REPLY, @@ -74,10 +75,10 @@ const createSocketAPI = ( logger: logger && createLogger('PHX', { level: 'debug' }), }); - wss.registerEvents('attempts:queue', { + wss.registerEvents('worker:queue', { [CLAIM]: (ws, event: PhoenixEvent) => { - const results = pullClaim(state, ws, event); - results.forEach((attempt) => { + const { attempts } = pullClaim(state, ws, event); + attempts.forEach((attempt) => { state.events.emit(CLAIM, { attemptId: attempt.id, payload: attempt, @@ -145,9 +146,10 @@ const createSocketAPI = ( const { queue } = state; let count = 1; + const attempts: CLAIM_ATTEMPT[] = []; const payload = { status: 'ok' as const, - response: [] as CLAIM_REPLY, + response: { attempts } as CLAIM_REPLY, }; while (count > 0 && queue.length) { @@ -156,13 +158,13 @@ const createSocketAPI = ( const next = queue.shift(); // TODO the token in the mock is trivial because we're not going to do any validation on it yet // TODO need to save the token associated with this attempt - payload.response.push({ id: next!, token: 'x.y.z' }); + attempts.push({ id: next!, token: 'x.y.z' }); count -= 1; startAttempt(next!); } - if (payload.response.length) { - logger?.info(`Claiming ${payload.response.length} attempts`); + if (attempts.length) { + logger?.info(`Claiming ${attempts.length} attempts`); } else { logger?.info('No claims (queue empty)'); } diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index f82f5ca32..67cba0313 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -1,11 +1,12 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; - +import Router from '@koa/router'; import { createMockLogger, Logger } from '@openfn/logger'; import createRestAPI from './api-rest'; import startWorkloop from './api/workloop'; +import claim from './api/claim'; import { execute } from './api/execute'; import joinAttemptChannel from './api/start-attempt'; import connectToLightning from './api/connect'; @@ -18,11 +19,13 @@ type ServerOptions = { port?: number; lightning?: string; // url to lightning instance logger?: Logger; + noLoop?: boolean; // disable the worker loop secret?: string; // worker secret }; function createServer(engine: any, options: ServerOptions = {}) { + console.log(options); const logger = options.logger || createMockLogger(); const port = options.port || 1234; @@ -47,6 +50,9 @@ function createServer(engine: any, options: ServerOptions = {}) { logger.info('Closing server'); }; + const router = new Router(); + app.use(router.routes()); + if (options.lightning) { logger.debug('Connecting to Lightning at', options.lightning); // TODO this is too hard to unit test, need to pull it out @@ -55,6 +61,7 @@ function createServer(engine: any, options: ServerOptions = {}) { logger.success('Connected to Lightning at', options.lightning); const startAttempt = async ({ id, token }: CLAIM_ATTEMPT) => { + // TODO need to verify the token against LIGHTNING_PUBLIC_KEY const { channel: attemptChannel, plan } = await joinAttemptChannel( socket, token, @@ -65,14 +72,33 @@ function createServer(engine: any, options: ServerOptions = {}) { }; logger.info('Starting workloop'); - // TODO maybe namespace the workloop logger differently? It's a bit annoying - startWorkloop(channel, startAttempt, logger, { - maxBackoff: options.maxBackoff, - }); + if (!options.noLoop) { + // TODO maybe namespace the workloop logger differently? It's a bit annoying + startWorkloop(channel, startAttempt, logger, { + maxBackoff: options.maxBackoff, + // timeout: 1000 * 60, // TMP debug poll once per minute + }); + } // debug/unit test API to run a workflow // TODO Only loads in dev mode? (app as any).execute = startAttempt; + + // Debug API to manually trigger a claim + router.post('/claim', async (ctx) => { + logger.info('triggering claim from POST request'); + return claim(channel, startAttempt, logger) + .then(() => { + logger.info('claim complete: attempt processed'); + ctx.body = 'complete'; + ctx.status = 200; + }) + .catch((e) => { + logger.info('claim complete: no attempts'); + ctx.body = 'no attempts'; + ctx.status = 204; + }); + }); }) .catch((e) => { logger.error( diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 63a2010d6..51d4c3546 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -13,6 +13,7 @@ type Args = { lightning?: string; repoDir?: string; secret?: string; + loop?: boolean; }; const args = yargs(hideBin(process.argv)) @@ -29,6 +30,7 @@ const args = yargs(hideBin(process.argv)) alias: 'l', description: 'Base url to Lightning websocket endpoint, eg, ws://locahost:4000/worker. Set to "mock" to use the default mock server', + default: 'http://localhost:4000/worker', }) .option('repo-dir', { alias: 'd', @@ -43,6 +45,11 @@ const args = yargs(hideBin(process.argv)) default: 'info', type: 'string', }) + .option('loop', { + description: 'Disable the claims loop', + default: true, + type: 'boolean', + }) .parse() as Args; const logger = createLogger('SRV', { level: args.log }); @@ -77,4 +84,5 @@ createWorker(engine, { lightning: args.lightning, logger, secret: args.secret, + noLoop: !args.loop, }); diff --git a/packages/ws-worker/test/api/workloop.test.ts b/packages/ws-worker/test/api/workloop.test.ts index d6c944278..caf11f560 100644 --- a/packages/ws-worker/test/api/workloop.test.ts +++ b/packages/ws-worker/test/api/workloop.test.ts @@ -23,6 +23,7 @@ test('workloop can be cancelled', async (t) => { [CLAIM]: () => { count++; cancel(); + return { attempts: [] }; }, }); @@ -40,6 +41,7 @@ test('workloop sends the attempts:claim event', (t) => { [CLAIM]: () => { t.pass(); done(); + return { attempts: [] }; }, }); cancel = startWorkloop(channel, () => {}, logger); @@ -57,6 +59,7 @@ test('workloop sends the attempts:claim event several times ', (t) => { t.pass(); done(); } + return { attempts: [] }; }, }); cancel = startWorkloop(channel, () => {}, logger); @@ -67,13 +70,13 @@ test('workloop calls execute if attempts:claim returns attempts', (t) => { return new Promise((done) => { let cancel; const channel = mockChannel({ - [CLAIM]: () => { - return [{ id: 'a' }]; - }, + [CLAIM]: () => ({ + attempts: [{ id: 'a', token: 'x.y.z' }], + }), }); const execute = (attempt) => { - t.deepEqual(attempt, { id: 'a' }); + t.deepEqual(attempt, { id: 'a', token: 'x.y.z' }); t.pass(); done(); }; diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index 25e994438..7d5ab2383 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -163,12 +163,13 @@ test.serial( new Promise(async (done) => { t.is(server.getQueueLength(), 0); - const channel = await join('attempts:queue'); + const channel = await join('worker:queue'); // response is an array of attempt ids channel.push(CLAIM).receive('ok', (response) => { - t.assert(Array.isArray(response)); - t.is(response.length, 0); + const { attempts } = response; + t.assert(Array.isArray(attempts)); + t.is(attempts.length, 0); t.is(server.getQueueLength(), 0); done(); @@ -183,20 +184,14 @@ test.serial( server.enqueueAttempt(attempt1); t.is(server.getQueueLength(), 1); - // This uses a shared channel at all workers sit in - // They all yell from time to time to ask for work - // Lightning responds with an attempt id and server id (target) - // What if: - // a) each worker has its own channel, so claims are handed out privately - // b) we use the 'ok' status to return work in the response - // this b pattern is much nicer - const channel = await join('attempts:queue'); + const channel = await join('worker:queue'); // response is an array of attempt ids channel.push(CLAIM).receive('ok', (response) => { - t.truthy(response); - t.is(response.length, 1); - t.deepEqual(response[0], { id: 'attempt-1', token: 'x.y.z' }); + const { attempts } = response; + t.truthy(attempts); + t.is(attempts.length, 1); + t.deepEqual(attempts[0], { id: 'attempt-1', token: 'x.y.z' }); // ensure the server state has changed t.is(server.getQueueLength(), 0); diff --git a/packages/ws-worker/test/socket-client.js b/packages/ws-worker/test/socket-client.js index 83ab39e52..4756e54b5 100644 --- a/packages/ws-worker/test/socket-client.js +++ b/packages/ws-worker/test/socket-client.js @@ -10,7 +10,7 @@ const socket = new phx.Socket(endpoint); socket.onOpen(() => { console.log('socket open!'); - const channel = socket.channel('attempts:queue'); + const channel = socket.channel('worker:queue'); channel.join().receive('ok', () => { console.log('connected to attempts queue'); From 500923c5929562076dc60528e1eb6326b3203f78 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 3 Oct 2023 16:57:28 +0100 Subject: [PATCH 113/232] worker: ensure execute gets called after claiming --- packages/ws-worker/src/api/execute.ts | 3 +++ packages/ws-worker/src/api/start-attempt.ts | 17 ++++++++++++----- packages/ws-worker/src/server.ts | 6 +++--- packages/ws-worker/src/start.ts | 4 ++-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 5d49a9579..2eb323800 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -53,6 +53,7 @@ export function execute( plan: ExecutionPlan ) { return new Promise(async (resolve) => { + logger.info('execute..'); // TODO add proper logger (maybe channel, rtm and logger comprise a context object) // tracking state for this attempt const state: AttemptState = { @@ -99,7 +100,9 @@ export function execute( }; if (typeof plan.initialState === 'string') { + logger.info('loading dataclip ', plan.initialState); plan.initialState = await loadState(channel, plan.initialState); + logger.success('dataclip loaded'); } engine.execute(plan, resolvers); diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index 1f9e3b5e4..b53cd4f0d 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -21,17 +21,24 @@ const joinAttemptChannel = ( ) => { return new Promise<{ channel: Channel; plan: ExecutionPlan }>( (resolve, reject) => { + // TMP - lightning seems to be sending two responses to me + // just for now, I'm gonna gate the handling here + let didReceiveOk = false; + // TODO use proper logger const channelName = `attempt:${attemptId}`; logger.debug('connecting to ', channelName); const channel = socket.channel(channelName, { token }); channel .join() - .receive('ok', async () => { - logger.success(`connected to ${channelName}`); - const plan = await loadAttempt(channel); - logger.debug('converted attempt as execution plan:', plan); - resolve({ channel, plan }); + .receive('ok', async (e) => { + if (!didReceiveOk) { + didReceiveOk = true; + logger.success(`connected to ${channelName}`, e); + const plan = await loadAttempt(channel); + logger.debug('converted attempt as execution plan:', plan); + resolve({ channel, plan }); + } }) .receive('error', (err) => { logger.error(`error connecting to ${channelName}`, err); diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 67cba0313..acdc9686d 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -25,7 +25,6 @@ type ServerOptions = { }; function createServer(engine: any, options: ServerOptions = {}) { - console.log(options); const logger = options.logger || createMockLogger(); const port = options.port || 1234; @@ -88,8 +87,9 @@ function createServer(engine: any, options: ServerOptions = {}) { router.post('/claim', async (ctx) => { logger.info('triggering claim from POST request'); return claim(channel, startAttempt, logger) - .then(() => { - logger.info('claim complete: attempt processed'); + .then(({ id, token }) => { + logger.info('claim complete: 1 attempt claimed'); + startAttempt({ id, token }); ctx.body = 'complete'; ctx.status = 200; }) diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 51d4c3546..1acbe786f 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -76,8 +76,8 @@ if (args.lightning === 'mock') { // logger.debug('engine created'); // use the mock rtm for now -const engine = createMockRTE('rtm'); -logger.debug('Mock RTM created'); +const engine = createMockRTE('rte'); +logger.debug('Mock RTE created'); createWorker(engine, { port: args.port, From 0213ec2bec3fc7531800378bbcd7f2b74ecd1e88 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 4 Oct 2023 10:20:38 +0100 Subject: [PATCH 114/232] sketching --- packages/ws-worker/src/api/channels/attempts.ts | 5 +++++ packages/ws-worker/src/api/channels/workers.ts | 3 +++ packages/ws-worker/src/api/execute.ts | 1 + 3 files changed, 9 insertions(+) create mode 100644 packages/ws-worker/src/api/channels/attempts.ts create mode 100644 packages/ws-worker/src/api/channels/workers.ts diff --git a/packages/ws-worker/src/api/channels/attempts.ts b/packages/ws-worker/src/api/channels/attempts.ts new file mode 100644 index 000000000..6075242c9 --- /dev/null +++ b/packages/ws-worker/src/api/channels/attempts.ts @@ -0,0 +1,5 @@ +// all the attempt stuff goes here +// join the attempt channel +// do we pull the attempt data here? I don't think so really tbh +// it shoud prob be in execute, which is kinda annoying ebcause executeis designed to take a plan +// i guess it just takes a cahannel and it pulls diff --git a/packages/ws-worker/src/api/channels/workers.ts b/packages/ws-worker/src/api/channels/workers.ts new file mode 100644 index 000000000..17d003acd --- /dev/null +++ b/packages/ws-worker/src/api/channels/workers.ts @@ -0,0 +1,3 @@ +// todo all the worker channel stuff goes here +// connect to worker +// handle auth diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 2eb323800..a957cfdad 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -50,6 +50,7 @@ export function execute( channel: Channel, engine: any, // TODO typing! logger: Logger, + // TODO firsdt thing we'll do here is pull the plan plan: ExecutionPlan ) { return new Promise(async (resolve) => { From 0d76a2ce3ca4644ef4f7d48cda3779b874ce0fac Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 4 Oct 2023 16:43:37 +0100 Subject: [PATCH 115/232] worker: replace pheonix-channels with pheonix --- packages/ws-worker/package.json | 4 +- packages/ws-worker/src/api/connect.ts | 10 +- packages/ws-worker/src/api/execute.ts | 9 +- packages/ws-worker/src/api/start-attempt.ts | 6 +- .../src/mock/lightning/api-sockets.ts | 54 +++++++---- .../src/mock/lightning/socket-server.ts | 96 ++++++++++++------- packages/ws-worker/src/types.d.ts | 5 +- packages/ws-worker/test/api/execute.test.ts | 6 +- packages/ws-worker/test/integration.test.ts | 50 +++++----- .../ws-worker/test/mock/lightning.test.ts | 37 +++---- .../ws-worker/test/mock/socket-server.test.ts | 45 +++++---- packages/ws-worker/test/socket-client.js | 30 ------ 12 files changed, 189 insertions(+), 163 deletions(-) delete mode 100644 packages/ws-worker/test/socket-client.js diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index a8eba7bac..3ff8bed83 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -18,9 +18,9 @@ "license": "ISC", "dependencies": { "@koa/router": "^12.0.0", + "@openfn/engine-multi": "workspace:*", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", - "@openfn/engine-multi": "workspace:*", "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", @@ -28,7 +28,7 @@ "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", "koa-logger": "^3.2.1", - "phoenix-channels": "^1.0.0", + "phoenix": "^1.7.7", "ws": "^8.14.1" }, "devDependencies": { diff --git a/packages/ws-worker/src/api/connect.ts b/packages/ws-worker/src/api/connect.ts index 913126e56..3a9d61561 100644 --- a/packages/ws-worker/src/api/connect.ts +++ b/packages/ws-worker/src/api/connect.ts @@ -1,4 +1,5 @@ -import phx from 'phoenix-channels'; +import { Socket as PhxSocket } from 'phoenix'; +import { WebSocket } from 'ws'; import generateWorkerToken from '../util/worker-token'; import type { Socket, Channel } from '../types'; @@ -11,14 +12,17 @@ export const connectToLightning = ( endpoint: string, serverId: string, secret: string, - SocketConstructor: Socket = phx.Socket + SocketConstructor: Socket = PhxSocket ) => { return new Promise(async (done, reject) => { // TODO does this token need to be fed back anyhow? // I think it's just used to connect and then forgotten? // If we reconnect we need a new token I guess? const token = await generateWorkerToken(secret, serverId); - const socket = new SocketConstructor(endpoint, { params: { token } }); + const socket = new SocketConstructor(endpoint, { + params: { token }, + transport: WebSocket, + }); // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) // Do we infinitely try to reconnect? diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index a957cfdad..87f741429 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -96,14 +96,14 @@ export function execute( const resolvers = { credential: (id: string) => loadCredential(channel, id), - // dataclip: (id: string) => loadState(channel, id), + // dataclip: (id: string) => loadDataclip(channel, id), // TODO not supported right now }; if (typeof plan.initialState === 'string') { - logger.info('loading dataclip ', plan.initialState); - plan.initialState = await loadState(channel, plan.initialState); + plan.initialState = await loadDataclip(channel, plan.initialState); logger.success('dataclip loaded'); + logger.debug(plan.initialState); } engine.execute(plan, resolvers); @@ -167,6 +167,7 @@ export function onWorkflowComplete( channel .push(ATTEMPT_COMPLETE, { final_dataclip_id: state.result!, + status: 'success', // TODO }) .receive('ok', () => { onComplete(result); @@ -186,7 +187,7 @@ export function onJobLog({ channel, state }: Context, event: JSONLog) { channel.push(ATTEMPT_LOG, evt); } -export async function loadState(channel: Channel, stateId: string) { +export async function loadDataclip(channel: Channel, stateId: string) { const result = await getWithReply(channel, GET_DATACLIP, { id: stateId, }); diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index b53cd4f0d..d698c1fa9 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -1,8 +1,6 @@ -import phx from 'phoenix-channels'; - import convertAttempt from '../util/convert-attempt'; import { getWithReply } from '../util'; -import { Attempt, Channel } from '../types'; +import { Attempt, Channel, Socket } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; import { GET_ATTEMPT } from '../events'; @@ -14,7 +12,7 @@ import type { Logger } from '@openfn/logger'; // we can feedback to // Maybe we need a general errors channel const joinAttemptChannel = ( - socket: phx.Socket, + socket: Socket, token: string, attemptId: string, logger: Logger diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index daa0a2016..f034400d8 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -34,6 +34,7 @@ import { RUN_START, RUN_START_PAYLOAD, RUN_START_REPLY, + RUN_COMPLETE_REPLY, } from '../../events'; import type { Server } from 'http'; @@ -142,7 +143,7 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic } = evt; + const { ref, join_ref, topic } = evt; const { queue } = state; let count = 1; @@ -169,7 +170,7 @@ const createSocketAPI = ( logger?.info('No claims (queue empty)'); } - ws.reply({ ref, topic, payload }); + ws.reply({ ref, join_ref, topic, payload }); return payload.response; } @@ -178,12 +179,13 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic } = evt; + const { ref, join_ref, topic } = evt; const attemptId = extractAttemptId(topic); const response = state.attempts[attemptId]; ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', @@ -197,11 +199,12 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic, payload } = evt; + const { ref, join_ref, topic, payload } = evt; const response = state.credentials[payload.id]; // console.log(topic, event, response); ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', @@ -210,24 +213,34 @@ const createSocketAPI = ( }); } + // TODO this mock function is broken in the phoenix package update + // (I am not TOO worried, the actual integration works fine) function getDataclip( state: ServerState, ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic, payload } = evt; - const dataclip = state.dataclips[payload.id]; + const { ref, topic, join_ref } = evt; + const dataclip = state.dataclips[evt.payload.id]; // Send the data as an ArrayBuffer (our stringify function will do this) - const response = enc.encode(stringify(dataclip)); - + const data = enc.encode( + stringify({ + status: 'ok', + response: dataclip, + }) + ); + + // If I encode the status & response into the payload, + // then this will correctly encode as binary data + // But! the reply system doesn't seem to work because we don't get the response status when it's encoded + // seems kinda odd ws.reply({ ref, + join_ref, topic, - payload: { - status: 'ok', - response, - }, + // payload: data.buffer, + payload: { status: 'ok', response: {} }, }); } @@ -236,13 +249,14 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic, payload } = evt; + const { ref, join_ref, topic, payload } = evt; const { attempt_id: attemptId } = payload; state.pending[attemptId].logs.push(payload); ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', @@ -256,7 +270,7 @@ const createSocketAPI = ( evt: PhoenixEvent, attemptId: string ) { - const { ref, topic, payload } = evt; + const { ref, join_ref, topic, payload } = evt; const { dataclip } = payload; logger?.info('Completed attempt ', attemptId); @@ -267,12 +281,10 @@ const createSocketAPI = ( ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', - // TODO final dataclip id - // this is kind of awkward to work out - // we gotta sha every dataclip, find a match, then return }, }); } @@ -282,12 +294,13 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic } = evt; + const { ref, join_ref, topic } = evt; if (!state.dataclips) { state.dataclips = {}; } ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', @@ -300,7 +313,7 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, topic, payload } = evt; + const { ref, join_ref, topic, payload } = evt; const { output_dataclip_id, output_dataclip } = evt.payload; if (output_dataclip_id) { @@ -311,8 +324,9 @@ const createSocketAPI = ( } // be polite and acknowledge the event - ws.reply({ + ws.reply({ ref, + join_ref, topic, payload: { status: 'ok', diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts index 81cac0fa3..6df17ba1e 100644 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ b/packages/ws-worker/src/mock/lightning/socket-server.ts @@ -6,6 +6,7 @@ */ import { WebSocketServer, WebSocket } from 'ws'; import querystring from 'query-string'; +import { Serializer } from 'phoenix'; import { ATTEMPT_PREFIX, extractAttemptId } from './util'; import { ServerState } from './server'; @@ -15,6 +16,31 @@ import type { Logger } from '@openfn/logger'; type Topic = string; +// copied from pheonix/assets/j/pheonix/serializer.js - we could implement both encoders manually! +// encode(msg, callback){ +// if(msg.payload.constructor === ArrayBuffer){ +// return callback(this.binaryEncode(msg)) +// } else { +// let payload = [msg.join_ref, msg.ref, msg.topic, msg.event, msg.payload] +// return callback(JSON.stringify(payload)) +// } +// }, + +// decode(rawPayload, callback){ +// if(rawPayload.constructor === ArrayBuffer){ +// return callback(this.binaryDecode(rawPayload)) +// } else { +// let [join_ref, ref, topic, event, payload] = JSON.parse(rawPayload) +// return callback({join_ref, ref, topic, event, payload}) +// } +// }, + +const decoder = Serializer.decode.bind(Serializer); +const decode = (data: any) => new Promise((done) => decoder(data, done)); + +const encoder = Serializer.encode.bind(Serializer); +const encode = (data: any) => new Promise((done) => encoder(data, done)); + export type PhoenixEventStatus = 'ok' | 'error' | 'timeout'; // websocket with a couple of dev-friendly APIs @@ -28,6 +54,7 @@ export type PhoenixEvent

= { event: string; payload: P; ref: string; + join_ref: string; }; export type PhoenixReply = { @@ -37,6 +64,7 @@ export type PhoenixReply = { response?: R; }; ref: string; + join_ref: string; }; type EventHandler = (ws: DevSocket, event: PhoenixEvent) => void; @@ -85,17 +113,11 @@ function createServer({ } const events = { - // testing (TODO shouldn't this be in a specific channel?) - ping: (ws: DevSocket, { topic, ref }: PhoenixEvent) => { - ws.sendJSON({ - topic, - ref, - event: 'pong', - payload: {}, - }); - }, // When joining a channel, we need to send a chan_reply_{ref} message back to the socket - phx_join: (ws: DevSocket, { topic, ref, payload }: PhoenixEvent) => { + phx_join: ( + ws: DevSocket, + { topic, ref, payload, join_ref }: PhoenixEvent + ) => { let status: PhoenixEventStatus = 'ok'; let response = 'ok'; @@ -116,6 +138,7 @@ function createServer({ topic, payload: { status, response }, ref, + join_ref, }); }, }; @@ -139,53 +162,60 @@ function createServer({ return; } - ws.reply = ({ ref, topic, payload }: PhoenixReply) => { + ws.reply = async ({ + ref, + topic, + payload, + join_ref, + }: PhoenixReply) => { + // TODO only stringify payload if not a buffer logger?.debug(`<< [${topic}] chan_reply_${ref} ` + stringify(payload)); - ws.send( - stringify({ - event: `chan_reply_${ref}`, - ref, - topic, - payload, - }) - ); + const evt = await encode({ + event: `chan_reply_${ref}`, + ref, + join_ref, + topic, + payload, + }); + ws.send(evt); }; - ws.sendJSON = ({ event, ref, topic, payload }: PhoenixEvent) => { + ws.sendJSON = async ({ event, ref, topic, payload }: PhoenixEvent) => { logger?.debug(`<< [${topic}] ${event} ` + stringify(payload)); - ws.send( - stringify({ - event, - ref, - topic, - payload, - }) - ); + const evt = await encode({ + event, + ref, + topic, + payload: stringify(payload), // TODO do we stringify this? All of it? + }); + ws.send(evt); }; - ws.on('message', function (data: string) { - const evt = JSON.parse(data) as PhoenixEvent; + ws.on('message', async function (data: string) { + // decode the data + const evt = (await decode(data)) as PhoenixEvent; onMessage(evt); if (evt.topic) { // phx sends this info in each message - const { topic, event, payload, ref } = evt; + const { topic, event, payload, ref, join_ref } = evt; logger?.debug(`>> [${topic}] ${event} ${ref} :: ${stringify(payload)}`); if (events[event]) { // handle system/phoenix events - events[event](ws, { topic, payload, ref }); + events[event](ws, { topic, payload, ref, join_ref }); } else { // handle custom/user events if (channels[topic] && channels[topic].size) { channels[topic].forEach((fn) => { - fn(ws, { event, topic, payload, ref }); + fn(ws, { event, topic, payload, ref, join_ref }); }); } else { // This behaviour is just a convenience for unit tesdting ws.reply({ ref, + join_ref, topic, payload: { status: 'error', diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index ca0659872..9aa130741 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -1,5 +1,3 @@ -import phx from 'phoenix-channels'; - export type Credential = Record; export type State = { @@ -62,6 +60,7 @@ export declare class Socket { constructor(endpoint: string, options: { params: any }); onOpen(callback: () => void): void; connect(): void; + channel(channelName: string, params: any): Channel; } export type Channel = { @@ -69,7 +68,7 @@ export type Channel = { // TODO it would be super nice to infer the event from the payload push:

(event: string, payload?: P) => ReceiveHook; - join:

(event: string, payload?: P) => ReceiveHook; + join: () => ReceiveHook; }; // type RuntimeExecutionPlanID = string; diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 2085d7755..04f35b872 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -18,7 +18,7 @@ import { onWorkflowStart, onWorkflowComplete, AttemptState, - loadState, + loadDataclip, loadCredential, } from '../../src/api/execute'; import createMockRTE from '../../src/mock/runtime-engine'; @@ -271,7 +271,7 @@ test('workflowComplete should call onComplete with final dataclip', async (t) => }); // TODO what if an error? -test('loadState should fetch a dataclip', async (t) => { +test('loadDataclip should fetch a dataclip', async (t) => { const channel = mockChannel({ [GET_DATACLIP]: ({ id }) => { t.is(id, 'xyz'); @@ -279,7 +279,7 @@ test('loadState should fetch a dataclip', async (t) => { }, }); - const state = await loadState(channel, 'xyz'); + const state = await loadDataclip(channel, 'xyz'); t.deepEqual(state, { data: {} }); }); diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 045afef7b..b8705260d 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -62,30 +62,35 @@ test.serial( }); } ); -test.serial('should run an attempt which returns intial state', async (t) => { - return new Promise((done) => { - lng.addDataclip('x', { - route: 66, - }); - const attempt = { - id: 'attempt-1', - dataclip_id: 'x', - jobs: [ - { - body: 'whatever', - }, - ], - }; +// TODO this is failing because the dataclip mock is broken +test.serial.skip( + 'should run an attempt which returns intial state', + async (t) => { + return new Promise((done) => { + lng.addDataclip('x', { + route: 66, + }); - lng.waitForResult(attempt.id).then((result) => { - t.deepEqual(result, { route: 66 }); - done(); - }); + const attempt = { + id: 'attempt-2', + dataclip_id: 'x', + jobs: [ + { + body: 'whatever', + }, + ], + }; - lng.enqueueAttempt(attempt); - }); -}); + lng.waitForResult(attempt.id).then((result) => { + t.deepEqual(result, { route: 66 }); + done(); + }); + + lng.enqueueAttempt(attempt); + }); + } +); // A basic high level integration test to ensure the whole loop works // This checks the events received by the lightning websocket @@ -189,7 +194,8 @@ test.serial( } ); -test.serial( +// TODO this is failing because the dataclip mock is broken +test.serial.skip( `events: lightning should receive a ${e.GET_DATACLIP} event`, (t) => { return new Promise((done) => { diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index 7d5ab2383..505717d97 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -1,7 +1,8 @@ import test from 'ava'; -import createLightningServer, { API_PREFIX } from '../../src/mock/lightning'; +import createLightningServer from '../../src/mock/lightning'; -import phx from 'phoenix-channels'; +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; import { attempts, credentials, dataclips } from './data'; import { @@ -13,7 +14,7 @@ import { GET_CREDENTIAL, GET_DATACLIP, } from '../../src/events'; -import type { Attempt } from '../../src/types'; +import type { Attempt, Channel } from '../../src/types'; import { JSONLog } from '@openfn/logger'; const endpoint = 'ws://localhost:7777/worker'; @@ -31,7 +32,11 @@ test.before( // Note that we need a token to connect, but the mock here // doesn't (yet) do any validation on that token - client = new phx.Socket(endpoint, { params: { token: 'x.y.z' } }); + client = new Socket(endpoint, { + params: { token: 'x.y.z' }, + timeout: 1000 * 120, + transport: WebSocket, + }); client.onOpen(done); client.connect(); }) @@ -47,10 +52,7 @@ test.after(() => { const attempt1 = attempts['attempt-1']; -const join = ( - channelName: string, - params: any = {} -): Promise => +const join = (channelName: string, params: any = {}): Promise => new Promise((done, reject) => { const channel = client.channel(channelName, params); channel @@ -121,7 +123,7 @@ test.serial('provide a phoenix websocket at /worker', (t) => { test.serial('reject ws connections without a token', (t) => { return new Promise((done) => { // client should be connected before this test runs - const socket = new phx.Socket(endpoint); + const socket = new Socket(endpoint, { transport: WebSocket }); socket.onClose(() => { t.pass(); done(); @@ -144,19 +146,6 @@ test.serial('respond to channel join requests', (t) => { // TODO: only allow authorised workers to join workers // TODO: only allow authorised attemtps to join an attempt channel -test.serial('get a reply to a ping event', (t) => { - return new Promise(async (done) => { - const channel = await join('test'); - - channel.on('pong', (payload) => { - t.pass('message received'); - done(); - }); - - channel.push('ping'); - }); -}); - test.serial( 'claim attempt: reply for zero items if queue is empty', (t) => @@ -311,7 +300,9 @@ test.serial('get credential through the attempt channel', async (t) => { }); }); -test.serial('get dataclip through the attempt channel', async (t) => { +// Skipping because the handling of the dataclip is broken right now +// since updating the phoenix module +test.serial.skip('get dataclip through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addDataclip('d', dataclips['d']); diff --git a/packages/ws-worker/test/mock/socket-server.test.ts b/packages/ws-worker/test/mock/socket-server.test.ts index 62142b5e8..2f7e5552d 100644 --- a/packages/ws-worker/test/mock/socket-server.test.ts +++ b/packages/ws-worker/test/mock/socket-server.test.ts @@ -1,5 +1,6 @@ import test from 'ava'; -import phx from 'phoenix-channels'; +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; import createServer from '../../src/mock/lightning/socket-server'; @@ -12,16 +13,23 @@ const wait = (duration = 10) => setTimeout(resolve, duration); }); -test.beforeEach(() => { - messages = []; - // @ts-ignore I don't care about missing server options here - server = createServer({ onMessage: (evt) => messages.push(evt) }); +test.beforeEach( + () => + new Promise((done) => { + messages = []; + // @ts-ignore I don't care about missing server options here + server = createServer({ onMessage: (evt) => messages.push(evt) }); - socket = new phx.Socket('ws://localhost:8080', { - params: { token: 'x.y.z' }, - }); - socket.connect(); -}); + socket = new Socket('ws://localhost:8080', { + transport: WebSocket, + params: { token: 'x.y.z' }, + }); + + socket.onOpen(done); + + socket.connect(); + }) +); test.afterEach(() => { server.close(); @@ -31,12 +39,17 @@ test.serial('respond to connection join requests', async (t) => { return new Promise((resolve) => { const channel = socket.channel('x', {}); - channel.join().receive('ok', (resp) => { - t.is(resp, 'ok'); - - channel.push('hello'); - resolve(); - }); + channel + .join() + .receive('ok', (resp) => { + t.is(resp, 'ok'); + + channel.push('hello'); + resolve(); + }) + .receive('error', (e) => { + console.log(e); + }); }); }); diff --git a/packages/ws-worker/test/socket-client.js b/packages/ws-worker/test/socket-client.js deleted file mode 100644 index 4756e54b5..000000000 --- a/packages/ws-worker/test/socket-client.js +++ /dev/null @@ -1,30 +0,0 @@ -// this is a standalone test script -// run from the commandline ie `node test/socket-client.js` -import phx from 'phoenix-channels'; - -const endpoint = 'ws://localhost:8888/worker'; - -console.log('connecting to socket at ', endpoint); -const socket = new phx.Socket(endpoint); - -socket.onOpen(() => { - console.log('socket open!'); - - const channel = socket.channel('worker:queue'); - channel.join().receive('ok', () => { - console.log('connected to attempts queue'); - - channel.on('pong', () => { - console.log('received pong!'); - }); - - channel.push('ping'); - }); - - setInterval(() => { - console.log('requesting work...'); - channel.push('attempts:claim'); - }, 500); -}); - -socket.connect(); From 578685a047dde1119f108b527f782ea57f217e1d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 07:09:05 +0100 Subject: [PATCH 116/232] package lock --- pnpm-lock.yaml | 117 ++++--------------------------------------------- 1 file changed, 8 insertions(+), 109 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 747bf6e87..8daa9830e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -500,9 +500,9 @@ importers: koa-logger: specifier: ^3.2.1 version: 3.2.1 - phoenix-channels: - specifier: ^1.0.0 - version: 1.0.0 + phoenix: + specifier: ^1.7.7 + version: 1.7.7 ws: specifier: ^8.14.1 version: 8.14.1 @@ -1495,6 +1495,7 @@ packages: /@openfn/language-common@2.0.0-rc3: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} + bundledDependencies: [] /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} @@ -2401,14 +2402,6 @@ packages: ieee754: 1.2.1 dev: true - /bufferutil@4.0.7: - resolution: {integrity: sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==} - engines: {node: '>=6.14.2'} - requiresBuild: true - dependencies: - node-gyp-build: 4.6.1 - dev: false - /builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} dependencies: @@ -2919,13 +2912,6 @@ packages: dependencies: array-find-index: 1.0.2 - /d@1.0.1: - resolution: {integrity: sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==} - dependencies: - es5-ext: 0.10.62 - type: 1.2.0 - dev: false - /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -2947,6 +2933,7 @@ packages: optional: true dependencies: ms: 2.0.0 + dev: true /debug@3.2.7(supports-color@5.5.0): resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} @@ -3225,31 +3212,6 @@ packages: is-symbol: 1.0.4 dev: true - /es5-ext@0.10.62: - resolution: {integrity: sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==} - engines: {node: '>=0.10'} - requiresBuild: true - dependencies: - es6-iterator: 2.0.3 - es6-symbol: 3.1.3 - next-tick: 1.1.0 - dev: false - - /es6-iterator@2.0.3: - resolution: {integrity: sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==} - dependencies: - d: 1.0.1 - es5-ext: 0.10.62 - es6-symbol: 3.1.3 - dev: false - - /es6-symbol@3.1.3: - resolution: {integrity: sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==} - dependencies: - d: 1.0.1 - ext: 1.7.0 - dev: false - /esbuild-android-64@0.14.54: resolution: {integrity: sha512-Tz2++Aqqz0rJ7kYBfz+iqyE3QMycD4vk7LBRyWaAVFgFtQ/O8EJOnVmTOiDWYZ/uYzB4kvP+bqejYdVKzE5lAQ==} engines: {node: '>=12'} @@ -3866,12 +3828,6 @@ packages: - supports-color dev: true - /ext@1.7.0: - resolution: {integrity: sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==} - dependencies: - type: 2.7.2 - dev: false - /extend-shallow@2.0.1: resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} engines: {node: '>=0.10.0'} @@ -4863,10 +4819,6 @@ packages: has-symbols: 1.0.3 dev: true - /is-typedarray@1.0.0: - resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} - dev: false - /is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -5568,6 +5520,7 @@ packages: /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + dev: true /ms@2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} @@ -5622,10 +5575,6 @@ packages: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} - /next-tick@1.1.0: - resolution: {integrity: sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==} - dev: false - /no-case@3.0.4: resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} dependencies: @@ -5645,11 +5594,6 @@ packages: whatwg-url: 5.0.0 dev: false - /node-gyp-build@4.6.1: - resolution: {integrity: sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==} - hasBin: true - dev: false - /nodemon@2.0.19: resolution: {integrity: sha512-4pv1f2bMDj0Eeg/MhGqxrtveeQ5/G/UVe9iO6uTZzjnRluSA4PVWf8CW99LUPwGB3eNIA7zUFoP77YuI7hOc0A==} engines: {node: '>=8.10.0'} @@ -6099,12 +6043,8 @@ packages: through2: 2.0.5 dev: true - /phoenix-channels@1.0.0: - resolution: {integrity: sha512-NVanumwkjzxUptGKBAMQ1W9njWrJom61rNpcTvSho9Hs441Lv8AJElXdkbycX9fFccc6OJViVLhDL0L5U/HqMg==} - dependencies: - websocket: 1.0.34 - transitivePeerDependencies: - - supports-color + /phoenix@1.7.7: + resolution: {integrity: sha512-moAN6e4Z16x/x1nswUpnTR2v5gm7HsI7eluZ2YnYUUsBNzi3cY/5frmiJfXIEi877IQAafzTfp8hd6vEUMme+w==} dev: false /picocolors@1.0.0: @@ -7624,20 +7564,6 @@ packages: mime-types: 2.1.35 dev: false - /type@1.2.0: - resolution: {integrity: sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==} - dev: false - - /type@2.7.2: - resolution: {integrity: sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==} - dev: false - - /typedarray-to-buffer@3.1.5: - resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} - dependencies: - is-typedarray: 1.0.0 - dev: false - /typescript@4.6.4: resolution: {integrity: sha512-9ia/jWHIEbo49HfjrLGfKbZSuWo9iTMwXO+Ca3pRsSpbsMbc7/IU8NKdCZVRRBafVPGnoJeFL76ZOAA84I9fEg==} engines: {node: '>=4.2.0'} @@ -7761,14 +7687,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /utf-8-validate@5.0.10: - resolution: {integrity: sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==} - engines: {node: '>=6.14.2'} - requiresBuild: true - dependencies: - node-gyp-build: 4.6.1 - dev: false - /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} dev: true @@ -7833,20 +7751,6 @@ packages: engines: {node: '>=0.8.0'} dev: true - /websocket@1.0.34: - resolution: {integrity: sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ==} - engines: {node: '>=4.0.0'} - dependencies: - bufferutil: 4.0.7 - debug: 2.6.9 - es5-ext: 0.10.62 - typedarray-to-buffer: 3.1.5 - utf-8-validate: 5.0.10 - yaeti: 0.0.6 - transitivePeerDependencies: - - supports-color - dev: false - /well-known-symbols@2.0.0: resolution: {integrity: sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==} engines: {node: '>=6'} @@ -7976,11 +7880,6 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} - /yaeti@0.0.6: - resolution: {integrity: sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==} - engines: {node: '>=0.10.32'} - dev: false - /yallist@2.1.2: resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} dev: true From ed1b90387ccd6268012c700d541393bb31f5a442 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 11:53:59 +0100 Subject: [PATCH 117/232] worker: fix mock to handle dataclips --- .../src/mock/lightning/api-sockets.ts | 20 +++----- .../src/mock/lightning/socket-server.ts | 29 ++++------- packages/ws-worker/src/server.ts | 7 ++- packages/ws-worker/test/integration.test.ts | 49 +++++++++---------- .../ws-worker/test/mock/lightning.test.ts | 4 +- 5 files changed, 45 insertions(+), 64 deletions(-) diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index f034400d8..b7c641a21 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -224,23 +224,17 @@ const createSocketAPI = ( const dataclip = state.dataclips[evt.payload.id]; // Send the data as an ArrayBuffer (our stringify function will do this) - const data = enc.encode( - stringify({ - status: 'ok', - response: dataclip, - }) - ); - - // If I encode the status & response into the payload, - // then this will correctly encode as binary data - // But! the reply system doesn't seem to work because we don't get the response status when it's encoded - // seems kinda odd + const payload = { + status: 'ok', + response: enc.encode(stringify(dataclip)), + }; + ws.reply({ ref, join_ref, topic, - // payload: data.buffer, - payload: { status: 'ok', response: {} }, + // @ts-ignore + payload, }); } diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts index 6df17ba1e..0fc1f0391 100644 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ b/packages/ws-worker/src/mock/lightning/socket-server.ts @@ -16,30 +16,19 @@ import type { Logger } from '@openfn/logger'; type Topic = string; -// copied from pheonix/assets/j/pheonix/serializer.js - we could implement both encoders manually! -// encode(msg, callback){ -// if(msg.payload.constructor === ArrayBuffer){ -// return callback(this.binaryEncode(msg)) -// } else { -// let payload = [msg.join_ref, msg.ref, msg.topic, msg.event, msg.payload] -// return callback(JSON.stringify(payload)) -// } -// }, - -// decode(rawPayload, callback){ -// if(rawPayload.constructor === ArrayBuffer){ -// return callback(this.binaryDecode(rawPayload)) -// } else { -// let [join_ref, ref, topic, event, payload] = JSON.parse(rawPayload) -// return callback({join_ref, ref, topic, event, payload}) -// } -// }, - const decoder = Serializer.decode.bind(Serializer); const decode = (data: any) => new Promise((done) => decoder(data, done)); const encoder = Serializer.encode.bind(Serializer); -const encode = (data: any) => new Promise((done) => encoder(data, done)); +const encode = (data: any) => + new Promise((done) => { + if (data.payload?.response && data.payload.response instanceof Uint8Array) { + // special encoding logic if the payload is a buffer + // (we need to do this for dataclips) + data.payload.response = Array.from(data.payload.response); + } + encoder(data, done); + }); export type PhoenixEventStatus = 'ok' | 'error' | 'timeout'; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index acdc9686d..486fbe335 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -70,13 +70,18 @@ function createServer(engine: any, options: ServerOptions = {}) { execute(attemptChannel, engine, logger, plan); }; - logger.info('Starting workloop'); if (!options.noLoop) { + logger.info('Starting workloop'); // TODO maybe namespace the workloop logger differently? It's a bit annoying startWorkloop(channel, startAttempt, logger, { maxBackoff: options.maxBackoff, // timeout: 1000 * 60, // TMP debug poll once per minute }); + } else { + logger.info('Workloop not starting'); + logger.info('This server will not auto-pull work from lightning.'); + logger.info('You can manually claim by posting to /claim'); + logger.info(`curl -X POST http://locahost:${port}/claim`); } // debug/unit test API to run a workflow diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index b8705260d..ba0756c90 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -63,34 +63,30 @@ test.serial( } ); -// TODO this is failing because the dataclip mock is broken -test.serial.skip( - 'should run an attempt which returns intial state', - async (t) => { - return new Promise((done) => { - lng.addDataclip('x', { - route: 66, - }); - - const attempt = { - id: 'attempt-2', - dataclip_id: 'x', - jobs: [ - { - body: 'whatever', - }, - ], - }; +test.serial('should run an attempt which returns intial state', async (t) => { + return new Promise((done) => { + lng.addDataclip('x', { + route: 66, + }); - lng.waitForResult(attempt.id).then((result) => { - t.deepEqual(result, { route: 66 }); - done(); - }); + const attempt = { + id: 'attempt-2', + dataclip_id: 'x', + jobs: [ + { + body: 'whatever', + }, + ], + }; - lng.enqueueAttempt(attempt); + lng.waitForResult(attempt.id).then((result) => { + t.deepEqual(result, { route: 66 }); + done(); }); - } -); + + lng.enqueueAttempt(attempt); + }); +}); // A basic high level integration test to ensure the whole loop works // This checks the events received by the lightning websocket @@ -194,8 +190,7 @@ test.serial( } ); -// TODO this is failing because the dataclip mock is broken -test.serial.skip( +test.serial( `events: lightning should receive a ${e.GET_DATACLIP} event`, (t) => { return new Promise((done) => { diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index 505717d97..60ed0bda7 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -300,9 +300,7 @@ test.serial('get credential through the attempt channel', async (t) => { }); }); -// Skipping because the handling of the dataclip is broken right now -// since updating the phoenix module -test.serial.skip('get dataclip through the attempt channel', async (t) => { +test.serial('get dataclip through the attempt channel', async (t) => { return new Promise(async (done) => { server.startAttempt(attempt1.id); server.addDataclip('d', dataclips['d']); From d101cf0e36b3d77a0bf4e56e1be6226a28d76f63 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 12:11:52 +0100 Subject: [PATCH 118/232] worker: disable log for now --- packages/ws-worker/src/api/execute.ts | 2 +- packages/ws-worker/src/server.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 87f741429..408bd96d7 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -88,7 +88,7 @@ export function execute( addEvent('workflow-start', onWorkflowStart), addEvent('job-start', onJobStart), addEvent('job-complete', onJobComplete), - addEvent('log', onJobLog), + // addEvent('log', onJobLog), // This will also resolve the promise addEvent('workflow-complete', onWorkflowComplete) ); diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 486fbe335..38a1a9317 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -78,10 +78,12 @@ function createServer(engine: any, options: ServerOptions = {}) { // timeout: 1000 * 60, // TMP debug poll once per minute }); } else { + logger.info(); logger.info('Workloop not starting'); logger.info('This server will not auto-pull work from lightning.'); logger.info('You can manually claim by posting to /claim'); logger.info(`curl -X POST http://locahost:${port}/claim`); + logger.info(); } // debug/unit test API to run a workflow From 32fb068efb84dcbdeae777732493d1cf3ac7609a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 12:16:21 +0100 Subject: [PATCH 119/232] restore log --- packages/ws-worker/src/api/execute.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 408bd96d7..87f741429 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -88,7 +88,7 @@ export function execute( addEvent('workflow-start', onWorkflowStart), addEvent('job-start', onJobStart), addEvent('job-complete', onJobComplete), - // addEvent('log', onJobLog), + addEvent('log', onJobLog), // This will also resolve the promise addEvent('workflow-complete', onWorkflowComplete) ); From 1536c687a0a2fc8c51ba95e1b1a737e9cd508424 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 12:27:40 +0100 Subject: [PATCH 120/232] worker: fix start run id --- packages/ws-worker/src/api/execute.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 87f741429..b08af1936 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -118,7 +118,7 @@ export function onJobStart({ channel, state }: Context, event: any) { state.activeJob = event; channel.push(RUN_START, { - run_id: state.activeJob!, + run_id: state.activeRun!, job_id: state.activeJob!, // input_dataclip_id what about this guy? }); From 4272cdbdf64dd14386c4fc6c52b68d77e8db4f8e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 13:12:27 +0100 Subject: [PATCH 121/232] worker: update events, add debugging In this commit the run_start event includes a job id, but lightning returns that the job does not exist --- packages/ws-worker/src/api/execute.ts | 49 +++++++++++++------ packages/ws-worker/src/events.ts | 5 +- packages/ws-worker/src/mock/runtime-engine.ts | 31 +++++------- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index b08af1936..fa92080e1 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -34,7 +34,7 @@ export type AttemptState = { dataclips: Record; // final dataclip id - result?: string; + lastDataclipId?: string; }; type Context = { @@ -50,7 +50,7 @@ export function execute( channel: Channel, engine: any, // TODO typing! logger: Logger, - // TODO firsdt thing we'll do here is pull the plan + // TODO first thing we'll do here is pull the plan plan: ExecutionPlan ) { return new Promise(async (resolve) => { @@ -59,6 +59,9 @@ export function execute( // tracking state for this attempt const state: AttemptState = { plan, + // set the result data clip id (which needs renaming) + // to the initial state + lastDataclipId: plan.initialState as string | undefined, }; const context: Context = { channel, state, logger, onComplete: resolve }; @@ -101,6 +104,7 @@ export function execute( }; if (typeof plan.initialState === 'string') { + logger.debug('loading dataclip', plan.initialState); plan.initialState = await loadDataclip(channel, plan.initialState); logger.success('dataclip loaded'); logger.debug(plan.initialState); @@ -115,12 +119,15 @@ export function execute( export function onJobStart({ channel, state }: Context, event: any) { // generate a run id and write it to state state.activeRun = crypto.randomUUID(); - state.activeJob = event; - - channel.push(RUN_START, { + state.activeJob = event.jobId; + const evt = { run_id: state.activeRun!, job_id: state.activeJob!, - // input_dataclip_id what about this guy? + input_dataclip_id: state.lastDataclipId, + }; + console.log('>>', evt); + channel.push(RUN_START, evt).receive('error', (e) => { + console.error('run start error', e); }); } @@ -128,7 +135,7 @@ export function onJobComplete({ channel, state }: Context, event: any) { const dataclipId = crypto.randomUUID(); channel.push(RUN_COMPLETE, { - run_id: state.activeRun!, + run_id: event.jobId, job_id: state.activeJob!, output_dataclip_id: dataclipId, output_dataclip: stringify(event.state), @@ -145,7 +152,7 @@ export function onJobComplete({ channel, state }: Context, event: any) { // (taking into account branches and stuff) // The problem is that the runtime will return the object, not an id, // so we have a bit of a mapping problem - state.result = dataclipId; + state.lastDataclipId = dataclipId; delete state.activeRun; delete state.activeJob; @@ -162,11 +169,11 @@ export function onWorkflowComplete( { state, channel, onComplete }: Context, _event: WorkflowCompleteEvent ) { - const result = state.dataclips[state.result!]; + const result = state.dataclips[state.lastDataclipId!]; channel .push(ATTEMPT_COMPLETE, { - final_dataclip_id: state.result!, + final_dataclip_id: state.lastDataclipId!, status: 'success', // TODO }) .receive('ok', () => { @@ -175,16 +182,26 @@ export function onWorkflowComplete( } export function onJobLog({ channel, state }: Context, event: JSONLog) { - // we basically just forward the log to lightning - // but we also need to attach the log id - const evt: ATTEMPT_LOG_PAYLOAD = { - ...event, + console.log(event); + + // lightning-friendly log object + const newLog: ATTEMPT_LOG_PAYLOAD = { attempt_id: state.plan.id!, + message: event.message, + source: event.name, + timestamp: event.time || Date.now(), }; if (state.activeRun) { - evt.run_id = state.activeRun; + newLog.run_id = state.activeRun; } - channel.push(ATTEMPT_LOG, evt); + + // TODO wrap a push with standard error and ok handlers + // like, maybe we only log after hte log was sent? Or log once for sending and once for acknowledged (either way)? + channel + .push(ATTEMPT_LOG, newLog) + .receive('error', (e) => { + console.error('log error', e); + }); } export async function loadDataclip(channel: Channel, stateId: string) { diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 9e94ce6b2..1ed4112fd 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -36,8 +36,11 @@ export type ATTEMPT_COMPLETE_PAYLOAD = { export type ATTEMPT_COMPLETE_REPLY = undefined; export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time -export type ATTEMPT_LOG_PAYLOAD = JSONLog & { +export type ATTEMPT_LOG_PAYLOAD = { + message: Array; + timestamp: number; attempt_id: string; + source?: string; // namespace job_id?: string; run_id?: string; }; diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index af9b29aa6..f125c8be5 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -98,33 +98,28 @@ function createMock(serverId?: string) { await resolvers.credential(configuration); } + const info = (...message: any[]) => { + dispatch('log', { + workflowId, + message: message, + level: 'info', + times: Date.now(), + name: 'mock', + }); + }; + // Get the job details from lightning // start instantly and emit as it goes dispatch('job-start', { workflowId, jobId, runId }); - dispatch('log', { - workflowId, - jobId, - message: ['Running job ' + jobId], - level: 'info', - }); + info('Running job ' + jobId); let nextState = initialState; // Try and parse the expression as JSON, in which case we use it as the final state try { // @ts-ignore nextState = JSON.parse(expression); // What does this look like? Should be a logger object - dispatch('log', { - workflowId, - jobId, - message: ['Parsing expression as JSON state'], - level: 'info', - }); - dispatch('log', { - workflowId, - jobId, - message: [nextState], - level: 'info', - }); + info('Parsing expression as JSON state'); + info(nextState); } catch (e) { // Do nothing, it's fine nextState = initialState; From fb353f4333f8839ffa107093b753d420f508ce75 Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 5 Oct 2023 14:50:03 +0200 Subject: [PATCH 122/232] Tweak log params --- packages/ws-worker/src/mock/runtime-engine.ts | 4 ++-- packages/ws-worker/src/start.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index f125c8be5..520b5d222 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -103,8 +103,8 @@ function createMock(serverId?: string) { workflowId, message: message, level: 'info', - times: Date.now(), - name: 'mock', + timestamp: Date.now(), + name: 'mck', }); }; diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 1acbe786f..d2b6cc740 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -30,7 +30,7 @@ const args = yargs(hideBin(process.argv)) alias: 'l', description: 'Base url to Lightning websocket endpoint, eg, ws://locahost:4000/worker. Set to "mock" to use the default mock server', - default: 'http://localhost:4000/worker', + default: 'ws://localhost:4000/worker', }) .option('repo-dir', { alias: 'd', From 3f8ec8a43a1c93ef40da087b5cb7049b0df3abcd Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Thu, 5 Oct 2023 15:17:58 +0200 Subject: [PATCH 123/232] Add reason to Run complete payload --- packages/ws-worker/src/api/execute.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index fa92080e1..d478a731a 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -139,6 +139,7 @@ export function onJobComplete({ channel, state }: Context, event: any) { job_id: state.activeJob!, output_dataclip_id: dataclipId, output_dataclip: stringify(event.state), + reason: "success" // HAND WAVE, change me! }); if (!state.dataclips) { From a1be2692b800b4b09dfa3a7a1dbe571e07a731b1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 15:19:33 +0100 Subject: [PATCH 124/232] workers: refactor execute with better event handlers --- packages/ws-worker/src/api/execute.ts | 84 +++--- packages/ws-worker/src/events.ts | 5 +- packages/ws-worker/src/mock/sockets.ts | 27 +- packages/ws-worker/test/api/execute.test.ts | 317 +++++++++++--------- 4 files changed, 241 insertions(+), 192 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index fa92080e1..af2b55fbb 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -73,9 +73,16 @@ export function execute( // b) pass the contexdt object into the hander // c) log the event const addEvent = (eventName: string, handler: EventHandler) => { - const wrappedFn = (event: any) => { - logger.info(`${plan.id} :: ${eventName}`); - handler(context, event); + const wrappedFn = async (event: any) => { + try { + await handler(context, event); + logger.info(`${plan.id} :: ${eventName} :: OK`); + } catch (e: any) { + logger.error( + `${plan.id} :: ${eventName} :: ERR: ${e.message || e.toString()}` + ); + logger.error(e); + } }; return { [eventName]: wrappedFn, @@ -86,6 +93,10 @@ export function execute( // Eg wait for a large dataclip to upload back to lightning before starting the next job // should we actually defer exeuction, or just the reporting? // Does it matter if logs aren't sent back in order? + // There are some practical requirements + // like we can't post a log until the job start has been acknowledged by Lightning + // (ie until the run was created at lightning's end) + // that probably means we need to cache here rather than slow down the runtime? const listeners = Object.assign( {}, addEvent('workflow-start', onWorkflowStart), @@ -114,32 +125,36 @@ export function execute( }); } +// async/await wrapper to push to a channel +// TODO move into utils I think? +export const sendEvent = (channel: Channel, event: string, payload?: any) => + new Promise((resolve, reject) => { + channel + .push(event, payload) + .receive('error', reject) + .receive('timeout', reject) + .receive('ok', resolve); + }); + // TODO maybe move all event handlers into api/events/* export function onJobStart({ channel, state }: Context, event: any) { // generate a run id and write it to state state.activeRun = crypto.randomUUID(); state.activeJob = event.jobId; - const evt = { + + return sendEvent(channel, RUN_START, { run_id: state.activeRun!, job_id: state.activeJob!, input_dataclip_id: state.lastDataclipId, - }; - console.log('>>', evt); - channel.push(RUN_START, evt).receive('error', (e) => { - console.error('run start error', e); }); } export function onJobComplete({ channel, state }: Context, event: any) { const dataclipId = crypto.randomUUID(); - channel.push(RUN_COMPLETE, { - run_id: event.jobId, - job_id: state.activeJob!, - output_dataclip_id: dataclipId, - output_dataclip: stringify(event.state), - }); + const run_id = state.activeRun; + const job_id = state.activeJob; if (!state.dataclips) { state.dataclips = {}; @@ -156,52 +171,51 @@ export function onJobComplete({ channel, state }: Context, event: any) { delete state.activeRun; delete state.activeJob; + + return sendEvent(channel, RUN_COMPLETE, { + run_id, + job_id, + output_dataclip_id: dataclipId, + output_dataclip: stringify(event.state), + }); } export function onWorkflowStart( { channel }: Context, _event: WorkflowStartEvent ) { - channel.push(ATTEMPT_START); + return sendEvent(channel, ATTEMPT_START); } -export function onWorkflowComplete( +export async function onWorkflowComplete( { state, channel, onComplete }: Context, _event: WorkflowCompleteEvent ) { const result = state.dataclips[state.lastDataclipId!]; - channel - .push(ATTEMPT_COMPLETE, { - final_dataclip_id: state.lastDataclipId!, - status: 'success', // TODO - }) - .receive('ok', () => { - onComplete(result); - }); + await sendEvent(channel, ATTEMPT_COMPLETE, { + final_dataclip_id: state.lastDataclipId!, + status: 'success', // TODO + }); + + onComplete(result); } export function onJobLog({ channel, state }: Context, event: JSONLog) { - console.log(event); - // lightning-friendly log object - const newLog: ATTEMPT_LOG_PAYLOAD = { + const log: ATTEMPT_LOG_PAYLOAD = { attempt_id: state.plan.id!, message: event.message, source: event.name, + level: event.level, timestamp: event.time || Date.now(), }; + if (state.activeRun) { - newLog.run_id = state.activeRun; + log.run_id = state.activeRun; } - // TODO wrap a push with standard error and ok handlers - // like, maybe we only log after hte log was sent? Or log once for sending and once for acknowledged (either way)? - channel - .push(ATTEMPT_LOG, newLog) - .receive('error', (e) => { - console.error('log error', e); - }); + return sendEvent(channel, ATTEMPT_LOG, log); } export async function loadDataclip(channel: Channel, stateId: string) { diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 1ed4112fd..56a9a05f7 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -31,6 +31,7 @@ export type ATTEMPT_START_REPLY = void; // no payload export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats export type ATTEMPT_COMPLETE_PAYLOAD = { final_dataclip_id: string; + status: 'success' | 'fail' | 'crash' | 'timeout'; stats?: any; }; // TODO dataclip -> result? output_dataclip? export type ATTEMPT_COMPLETE_REPLY = undefined; @@ -40,15 +41,13 @@ export type ATTEMPT_LOG_PAYLOAD = { message: Array; timestamp: number; attempt_id: string; + level?: string; source?: string; // namespace job_id?: string; run_id?: string; }; export type ATTEMPT_LOG_REPLY = void; -// this should not happen - this is "could not execute" rather than "complete with errors" -export const ATTEMPT_ERROR = 'attempt:error'; - export const RUN_START = 'run:start'; export type RUN_START_PAYLOAD = { job_id: string; diff --git a/packages/ws-worker/src/mock/sockets.ts b/packages/ws-worker/src/mock/sockets.ts index fc1db1081..99a969a05 100644 --- a/packages/ws-worker/src/mock/sockets.ts +++ b/packages/ws-worker/src/mock/sockets.ts @@ -8,18 +8,31 @@ export const mockChannel = (callbacks: Record = {}) => { callbacks[event] = fn; }, push:

(event: string, payload?: P) => { + const responses = {} as Record<'ok' | 'error' | 'timeout', EventHandler>; + // if a callback was registered, trigger it // otherwise do nothing - let result: any; - if (callbacks[event]) { - result = callbacks[event](payload); - } + setTimeout(() => { + if (callbacks[event]) { + try { + const result = callbacks[event](payload); + responses.ok?.(result); + } catch (e) { + responses.error?.(e); + } + } + }, 1); - return { - receive: (_status: string, callback: EventHandler) => { - setTimeout(() => callback(result), 1); + const receive = { + receive: ( + status: 'ok' | 'error' | 'timeout' = 'ok', + callback: EventHandler + ) => { + responses[status] = callback; + return receive; }, }; + return receive; }, join: () => { if (callbacks.join) { diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 04f35b872..2b69c31f5 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -20,6 +20,7 @@ import { AttemptState, loadDataclip, loadCredential, + sendEvent, } from '../../src/api/execute'; import createMockRTE from '../../src/mock/runtime-engine'; import { mockChannel } from '../../src/mock/sockets'; @@ -30,6 +31,37 @@ const enc = new TextEncoder(); const toArrayBuffer = (obj: any) => enc.encode(stringify(obj)); +const noop = () => true; + +const mockEventHandlers = { + [ATTEMPT_START]: noop, + [RUN_START]: noop, + [ATTEMPT_LOG]: noop, + [RUN_COMPLETE]: noop, + [ATTEMPT_COMPLETE]: noop, +}; + +test('send event should resolve when the event is acknowledged', async (t) => { + const channel = mockChannel({ + echo: (x) => x, + }); + + const result = await sendEvent(channel, 'echo', 22); + t.is(result, 22); +}); + +test('send event should throw if the event errors', async (t) => { + const channel = mockChannel({ + echo: (x) => { + throw new Error('err'); + }, + }); + + await t.throwsAsync(() => sendEvent(channel, 'echo', 22), { + message: 'err', + }); +}); + test('jobStart should set a run id and active job on state', async (t) => { const plan = { id: 'attempt-1' }; const jobId = 'job-1'; @@ -38,34 +70,35 @@ test('jobStart should set a run id and active job on state', async (t) => { plan, } as AttemptState; - const channel = mockChannel({}); + const channel = mockChannel({ + [RUN_START]: (x) => x, + }); - onJobStart({ channel, state }, jobId); + await onJobStart({ channel, state }, { jobId }); t.is(state.activeJob, jobId); t.truthy(state.activeRun); }); test('jobStart should send a run:start event', async (t) => { - return new Promise((done) => { - const plan = { id: 'attempt-1' }; - const jobId = 'job-1'; - - const state = { - plan, - } as AttemptState; - - const channel = mockChannel({ - [RUN_START]: (evt) => { - t.is(evt.job_id, jobId); - t.truthy(evt.run_id); + const plan = { id: 'attempt-1' }; + const jobId = 'job-1'; - done(); - }, - }); + const state = { + plan, + lastDataclipId: 'abc', // this will be set to initial state by execute + } as AttemptState; - onJobStart({ channel, state }, jobId); + const channel = mockChannel({ + [RUN_START]: (evt) => { + t.is(evt.job_id, jobId); + t.is(evt.input_dataclip_id, state.lastDataclipId); + t.truthy(evt.run_id); + return true; + }, }); + + await onJobStart({ channel, state }, { jobId }); }); test('jobComplete should clear the run id and active job on state', async (t) => { @@ -78,17 +111,19 @@ test('jobComplete should clear the run id and active job on state', async (t) => activeRun: 'b', } as AttemptState; - const channel = mockChannel({}); + const channel = mockChannel({ + [RUN_COMPLETE]: () => true, + }); const event = { state: { x: 10 } }; - onJobComplete({ channel, state }, event); + await onJobComplete({ channel, state }, event); t.falsy(state.activeJob); t.falsy(state.activeRun); }); test('jobComplete should save the dataclip to state', async (t) => { - const plan = { id: 'attempt-1' }; + const plan = { id: 'attempt-1' } as ExecutionPlan; const jobId = 'job-1'; const state = { @@ -99,10 +134,12 @@ test('jobComplete should save the dataclip to state', async (t) => { result: undefined, } as AttemptState; - const channel = mockChannel({}); + const channel = mockChannel({ + [RUN_COMPLETE]: () => true, + }); const event = { state: { x: 10 } }; - onJobComplete({ channel, state }, event); + await onJobComplete({ channel, state }, event); t.is(Object.keys(state.dataclips).length, 1); const [dataclip] = Object.values(state.dataclips); @@ -110,164 +147,148 @@ test('jobComplete should save the dataclip to state', async (t) => { }); test('jobComplete should send a run:complete event', async (t) => { - return new Promise((done) => { - const plan = { id: 'attempt-1' }; - const jobId = 'job-1'; - const result = { x: 10 }; - - const state = { - plan, - activeJob: jobId, - activeRun: 'b', - } as AttemptState; - - const channel = mockChannel({ - [RUN_COMPLETE]: (evt) => { - t.is(evt.job_id, jobId); - t.truthy(evt.run_id); - t.truthy(evt.output_dataclip_id); - t.is(evt.output_dataclip, JSON.stringify(result)); - - done(); - }, - }); + const plan = { id: 'attempt-1' }; + const jobId = 'job-1'; + const result = { x: 10 }; - const event = { state: result }; - onJobComplete({ channel, state }, event); + const state = { + plan, + activeJob: jobId, + activeRun: 'b', + } as AttemptState; + + const channel = mockChannel({ + [RUN_COMPLETE]: (evt) => { + t.is(evt.job_id, jobId); + t.truthy(evt.run_id); + t.truthy(evt.output_dataclip_id); + t.is(evt.output_dataclip, JSON.stringify(result)); + }, }); + + const event = { state: result }; + await onJobComplete({ channel, state }, event); }); test('jobLog should should send a log event outside a run', async (t) => { - return new Promise((done) => { - const plan = { id: 'attempt-1' }; - - const log: JSONLog = { - name: 'R/T', - level: 'info', - time: new Date().getTime(), - message: ['ping'], - }; + const plan = { id: 'attempt-1' }; - const result = { - ...log, - attempt_id: plan.id, - }; + const log: JSONLog = { + name: 'R/T', + level: 'info', + time: Date.now(), + message: ['ping'], + }; - const state = { - plan, - // No active run - } as AttemptState; + const result = { + attempt_id: plan.id, + message: log.message, + timestamp: log.time, + level: log.level, + source: log.name, + }; - const channel = mockChannel({ - [ATTEMPT_LOG]: (evt) => { - t.deepEqual(evt, result); - done(); - }, - }); + const state = { + plan, + // No active run + } as AttemptState; - onJobLog({ channel, state }, log); + const channel = mockChannel({ + [ATTEMPT_LOG]: (evt) => { + t.deepEqual(evt, result); + }, }); + + await onJobLog({ channel, state }, log); }); test('jobLog should should send a log event inside a run', async (t) => { - return new Promise((done) => { - const plan = { id: 'attempt-1' }; - const jobId = 'job-1'; - - const log: JSONLog = { - name: 'R/T', - level: 'info', - time: new Date().getTime(), - message: ['ping'], - }; + const plan = { id: 'attempt-1' }; + const jobId = 'job-1'; - const state = { - plan, - activeJob: jobId, - activeRun: 'b', - } as AttemptState; - - const channel = mockChannel({ - [ATTEMPT_LOG]: (evt) => { - t.truthy(evt.run_id); - t.deepEqual(evt.message, log.message); - t.is(evt.level, log.level); - t.is(evt.name, log.name); - t.is(evt.time, log.time); - done(); - }, - }); + const log: JSONLog = { + name: 'R/T', + level: 'info', + time: new Date().getTime(), + message: ['ping'], + }; - onJobLog({ channel, state }, log); + const state = { + plan, + activeJob: jobId, + activeRun: 'b', + } as AttemptState; + + const channel = mockChannel({ + [ATTEMPT_LOG]: (evt) => { + t.truthy(evt.run_id); + t.deepEqual(evt.message, log.message); + t.is(evt.level, log.level); + t.is(evt.source, log.name); + t.is(evt.timestamp, log.time); + }, }); + + await onJobLog({ channel, state }, log); }); test('workflowStart should send an empty attempt:start event', async (t) => { - return new Promise((done) => { - const channel = mockChannel({ - [ATTEMPT_START]: () => { - t.pass(); - - done(); - }, - }); - - onWorkflowStart({ channel }); + const channel = mockChannel({ + [ATTEMPT_START]: () => { + t.pass(); + }, }); + + await onWorkflowStart({ channel }); }); test('workflowComplete should send an attempt:complete event', async (t) => { - return new Promise((done) => { - const result = { answer: 42 }; - - const state = { - dataclips: { - x: result, - }, - result: 'x', - }; + const result = { answer: 42 }; - const channel = mockChannel({ - [ATTEMPT_COMPLETE]: (evt) => { - t.deepEqual(evt.final_dataclip_id, 'x'); + const state = { + dataclips: { + x: result, + }, + lastDataclipId: 'x', + }; - done(); - }, - }); + const channel = mockChannel({ + [ATTEMPT_COMPLETE]: (evt) => { + t.deepEqual(evt.final_dataclip_id, 'x'); + }, + }); - const event = {}; + const event = {}; - const context = { channel, state, onComplete: () => {} }; - onWorkflowComplete(context, event); - }); + const context = { channel, state, onComplete: () => {} }; + await onWorkflowComplete(context, event); }); test('workflowComplete should call onComplete with final dataclip', async (t) => { - return new Promise((done) => { - const result = { answer: 42 }; + const result = { answer: 42 }; - const state = { - dataclips: { - x: result, - }, - result: 'x', - }; + const state = { + dataclips: { + x: result, + }, + lastDataclipId: 'x', + }; - const channel = mockChannel(); + const channel = mockChannel({ + [ATTEMPT_COMPLETE]: () => true, + }); - const context = { - channel, - state, - onComplete: (finalState) => { - t.deepEqual(result, finalState); - done(); - }, - }; + const context = { + channel, + state, + onComplete: (finalState) => { + t.deepEqual(result, finalState); + }, + }; - const event = { state: result }; + const event = { state: result }; - onWorkflowComplete(context, event); - }); + await onWorkflowComplete(context, event); }); // TODO what if an error? @@ -297,7 +318,7 @@ test('loadCredential should fetch a credential', async (t) => { }); test('execute should return the final result', async (t) => { - const channel = mockChannel(); + const channel = mockChannel(mockEventHandlers); const engine = createMockRTE(); const logger = createMockLogger(); @@ -321,6 +342,7 @@ test('execute should lazy-load a credential', async (t) => { let didCallCredentials = false; const channel = mockChannel({ + ...mockEventHandlers, [GET_CREDENTIAL]: (id) => { t.truthy(id); didCallCredentials = true; @@ -349,6 +371,7 @@ test('execute should lazy-load initial state', async (t) => { let didCallState = false; const channel = mockChannel({ + ...mockEventHandlers, [GET_DATACLIP]: (id) => { t.truthy(id); didCallState = true; From 4a96b8a72724a1df407cced45802e071d33ad7c9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 15:31:48 +0100 Subject: [PATCH 125/232] worker: handle trigger nodes better --- packages/ws-worker/src/mock/runtime-engine.ts | 8 +++- .../test/mock/runtime-engine.test.ts | 38 ++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 520b5d222..45cacfb83 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -87,7 +87,13 @@ function createMock(serverId?: string) { initialState = {}, resolvers: LazyResolvers = mockResolvers ) => { - const { id, expression, configuration } = job; + const { id, expression, configuration, adaptor } = job; + + // If no expression or adaptor, this is (probably) a trigger node. + // Silently do nothing + if (!expression && !adaptor) { + return initialState; + } const runId = crypto.randomUUID(); diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index a19aa037b..3f933ea91 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -220,4 +220,40 @@ test('only listen to events for the correct workflow', async (t) => { t.pass(); }); -// test('inital dataclip') +test('do nothing for a job if no expression and adaptor (trigger node)', async (t) => { + const workflow = { + id: 'w1', + jobs: [ + { + id: 'j1', + }, + ], + } as ExecutionPlan; + + const engine = create(); + + let didCallEvent = false; + + engine.listen(workflow.id, { + 'job-start': () => { + didCallEvent = true; + }, + 'job-complete': () => { + didCallEvent = true; + }, + log: () => { + didCallEvent = true; + }, + 'workflow-start': () => { + // this can be called + }, + 'workflow-complete': () => { + // and this + }, + }); + + engine.execute(workflow); + await waitForEvent(engine, 'workflow-complete'); + + t.false(didCallEvent); +}); From c2c473f6eed1ff465872eb4eb76ea580848d0fed Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 16:56:33 +0100 Subject: [PATCH 126/232] worker: tweak output --- packages/ws-worker/src/server.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 38a1a9317..2d77bdd44 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -78,12 +78,12 @@ function createServer(engine: any, options: ServerOptions = {}) { // timeout: 1000 * 60, // TMP debug poll once per minute }); } else { - logger.info(); - logger.info('Workloop not starting'); + logger.break(); + logger.warn('Workloop not starting'); logger.info('This server will not auto-pull work from lightning.'); - logger.info('You can manually claim by posting to /claim'); - logger.info(`curl -X POST http://locahost:${port}/claim`); - logger.info(); + logger.info('You can manually claim by posting to /claim, eg:'); + logger.info(` curl -X POST http://locahost:${port}/claim`); + logger.break(); } // debug/unit test API to run a workflow From f9864caf5887f8efda2ba33feaffe5df286aecf3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 18:03:02 +0100 Subject: [PATCH 127/232] worker: type fixes and fixed double connect issue --- packages/ws-worker/package.json | 6 ++- packages/ws-worker/src/api-rest.ts | 30 ------------ .../api/channels/{attempts.ts => attempts} | 2 + .../src/api/channels/{workers.ts => workers} | 0 packages/ws-worker/src/api/claim.ts | 19 ++++---- packages/ws-worker/src/api/connect.ts | 7 +-- packages/ws-worker/src/api/execute.ts | 32 +++++++++---- packages/ws-worker/src/api/start-attempt.ts | 4 +- packages/ws-worker/src/events.ts | 1 - .../ws-worker/src/mock/lightning/api-dev.ts | 42 ++++++++++++---- .../src/mock/lightning/api-sockets.ts | 13 +++-- .../ws-worker/src/mock/lightning/server.ts | 6 +-- .../src/mock/lightning/socket-server.ts | 7 ++- packages/ws-worker/src/mock/runtime-engine.ts | 7 +-- packages/ws-worker/src/server.ts | 8 +--- packages/ws-worker/src/start.ts | 3 +- packages/ws-worker/src/types.d.ts | 26 +++++----- .../ws-worker/test/mock/lightning.test.ts | 38 ++++++++------- packages/ws-worker/tsconfig.json | 2 +- pnpm-lock.yaml | 48 ++++++++++++++++--- 20 files changed, 180 insertions(+), 121 deletions(-) delete mode 100644 packages/ws-worker/src/api-rest.ts rename packages/ws-worker/src/api/channels/{attempts.ts => attempts} (94%) rename packages/ws-worker/src/api/channels/{workers.ts => workers} (100%) diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 3ff8bed83..786e9212a 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -34,8 +34,12 @@ "devDependencies": { "@types/koa": "^2.13.5", "@types/koa-bodyparser": "^4.3.10", - "@types/koa-router": "^7.4.4", + "@types/koa-route": "^3.2.6", + "@types/koa-websocket": "^5.0.8", + "@types/koa__router": "^12.0.1", "@types/node": "^18.15.3", + "@types/nodemon": "1.19.3", + "@types/phoenix": "^1.6.2", "@types/yargs": "^17.0.12", "ava": "5.1.0", "koa-route": "^3.2.0", diff --git a/packages/ws-worker/src/api-rest.ts b/packages/ws-worker/src/api-rest.ts deleted file mode 100644 index febd9059f..000000000 --- a/packages/ws-worker/src/api-rest.ts +++ /dev/null @@ -1,30 +0,0 @@ -// @ts-ignore -import Router from '@koa/router'; -import type { Logger } from '@openfn/logger'; - -import healthcheck from './middleware/healthcheck'; -import workflow from './middleware/workflow'; - -// this defines the main API routes in a nice central place - -// So what is the API of this server? -// It's mostly a pull model, apart from I think the healthcheck -// Should have diagnostic and reporting APIs -// maybe even a simple frontend? - -const createAPI = (app: any, logger: Logger) => { - const router = new Router(); - - router.get('/healthcheck', healthcheck); - - // Dev API to run a workflow - // This is totally wrong now - // router.post('/workflow', workflow(execute, logger)); - - app.use(router.routes()); - app.use(router.allowedMethods()); - - return router; -}; - -export default createAPI; diff --git a/packages/ws-worker/src/api/channels/attempts.ts b/packages/ws-worker/src/api/channels/attempts similarity index 94% rename from packages/ws-worker/src/api/channels/attempts.ts rename to packages/ws-worker/src/api/channels/attempts index 6075242c9..39c089914 100644 --- a/packages/ws-worker/src/api/channels/attempts.ts +++ b/packages/ws-worker/src/api/channels/attempts @@ -1,3 +1,5 @@ +// @ts-ignore + // all the attempt stuff goes here // join the attempt channel // do we pull the attempt data here? I don't think so really tbh diff --git a/packages/ws-worker/src/api/channels/workers.ts b/packages/ws-worker/src/api/channels/workers similarity index 100% rename from packages/ws-worker/src/api/channels/workers.ts rename to packages/ws-worker/src/api/channels/workers diff --git a/packages/ws-worker/src/api/claim.ts b/packages/ws-worker/src/api/claim.ts index 65a9fe84f..5358a7138 100644 --- a/packages/ws-worker/src/api/claim.ts +++ b/packages/ws-worker/src/api/claim.ts @@ -16,12 +16,11 @@ const claim = ( channel .push(CLAIM, { demand: 1 }) .receive('ok', ({ attempts }: CLAIM_REPLY) => { - logger.debug('pull ok', attempts); + logger.debug(`pulled ${attempts.length} attempts`); // TODO what if we get here after we've been cancelled? // the events have already been claimed... if (!attempts?.length) { - logger.debug('no attempts, backing off'); // throw to backoff and try again return reject(new Error('claim failed')); } @@ -31,15 +30,15 @@ const claim = ( execute(attempt); resolve(); }); - }) - // TODO need implementations for both of these really - // What do we do if we fail to join the worker channel? - .receive('error', (r) => { - logger.debug('pull err'); - }) - .receive('timeout', (r) => { - logger.debug('pull timeout'); }); + // // TODO need implementations for both of these really + // // What do we do if we fail to join the worker channel? + // .receive('error', () => { + // logger.debug('pull err'); + // }) + // .receive('timeout', () => { + // logger.debug('pull timeout'); + // }); }); }; diff --git a/packages/ws-worker/src/api/connect.ts b/packages/ws-worker/src/api/connect.ts index 3a9d61561..5028659b4 100644 --- a/packages/ws-worker/src/api/connect.ts +++ b/packages/ws-worker/src/api/connect.ts @@ -12,13 +12,14 @@ export const connectToLightning = ( endpoint: string, serverId: string, secret: string, - SocketConstructor: Socket = PhxSocket + SocketConstructor = PhxSocket ) => { return new Promise(async (done, reject) => { // TODO does this token need to be fed back anyhow? // I think it's just used to connect and then forgotten? // If we reconnect we need a new token I guess? const token = await generateWorkerToken(secret, serverId); + // @ts-ignore ts doesn't like the constructor here at all const socket = new SocketConstructor(endpoint, { params: { token }, transport: WebSocket, @@ -31,7 +32,7 @@ export const connectToLightning = ( socket.onOpen(() => { // join the queue channel // TODO should this send the worker token? - const channel = socket.channel('worker:queue'); + const channel = socket.channel('worker:queue') as Channel; channel .join() @@ -47,7 +48,7 @@ export const connectToLightning = ( }); // TODO what even happens if the connection fails? - socket.onError((e) => { + socket.onError((e: any) => { reject(e); }); diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 91beb9039..ba94af8cd 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -9,7 +9,6 @@ import { ATTEMPT_START_PAYLOAD, GET_CREDENTIAL, GET_DATACLIP, - GET_DATACLIP_PAYLOAD, RUN_COMPLETE, RUN_COMPLETE_PAYLOAD, RUN_START, @@ -44,6 +43,15 @@ type Context = { onComplete: (result: any) => void; }; +// mapping engine events to lightning events +const eventMap = { + 'workflow-start': ATTEMPT_START, + 'job-start': RUN_START, + 'job-complete': RUN_COMPLETE, + log: ATTEMPT_LOG, + 'workflow-complete': ATTEMPT_COMPLETE, +}; + // pass a web socket connected to the attempt channel // this thing will do all the work export function execute( @@ -55,31 +63,35 @@ export function execute( ) { return new Promise(async (resolve) => { logger.info('execute..'); - // TODO add proper logger (maybe channel, rtm and logger comprise a context object) - // tracking state for this attempt + const state: AttemptState = { plan, // set the result data clip id (which needs renaming) // to the initial state lastDataclipId: plan.initialState as string | undefined, + dataclips: {}, }; const context: Context = { channel, state, logger, onComplete: resolve }; type EventHandler = (context: any, event: any) => void; - // Utility funciton to - // a) bind an event handler to a event - // b) pass the contexdt object into the hander - // c) log the event + // Utility function to: + // a) bind an event handler to a runtime-engine event + // b) pass the context object into the hander + // c) log the response from the websocket from lightning const addEvent = (eventName: string, handler: EventHandler) => { const wrappedFn = async (event: any) => { + // @ts-ignore + const lightningEvent = eventMap[eventName]; try { await handler(context, event); - logger.info(`${plan.id} :: ${eventName} :: OK`); + logger.info(`${plan.id} :: ${lightningEvent} :: OK`); } catch (e: any) { logger.error( - `${plan.id} :: ${eventName} :: ERR: ${e.message || e.toString()}` + `${plan.id} :: ${lightningEvent} :: ERR: ${ + e.message || e.toString() + }` ); logger.error(e); } @@ -132,7 +144,7 @@ export const sendEvent = (channel: Channel, event: string, payload?: any) => channel .push(event, payload) .receive('error', reject) - .receive('timeout', reject) + .receive('timeout', () => reject(new Error('timeout'))) .receive('ok', resolve); }); diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index d698c1fa9..c6e96ba20 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -29,7 +29,7 @@ const joinAttemptChannel = ( const channel = socket.channel(channelName, { token }); channel .join() - .receive('ok', async (e) => { + .receive('ok', async (e: any) => { if (!didReceiveOk) { didReceiveOk = true; logger.success(`connected to ${channelName}`, e); @@ -38,7 +38,7 @@ const joinAttemptChannel = ( resolve({ channel, plan }); } }) - .receive('error', (err) => { + .receive('error', (err: any) => { logger.error(`error connecting to ${channelName}`, err); reject(err); }); diff --git a/packages/ws-worker/src/events.ts b/packages/ws-worker/src/events.ts index 56a9a05f7..ba8f5b97c 100644 --- a/packages/ws-worker/src/events.ts +++ b/packages/ws-worker/src/events.ts @@ -1,4 +1,3 @@ -import { JSONLog } from '@openfn/logger'; import { Attempt } from './types'; // track socket event names as constants to keep refactoring easier diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts index 43a327b83..a0e3fad65 100644 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ b/packages/ws-worker/src/mock/lightning/api-dev.ts @@ -4,7 +4,7 @@ */ import Koa from 'koa'; import Router from '@koa/router'; -import { JSONLog, Logger } from '@openfn/logger'; +import { Logger } from '@openfn/logger'; import crypto from 'node:crypto'; import { Attempt } from '../../types'; @@ -13,18 +13,41 @@ import { ATTEMPT_COMPLETE, ATTEMPT_COMPLETE_PAYLOAD } from '../../events'; type LightningEvents = 'log' | 'attempt-complete'; +type DataClip = any; + export type DevApp = Koa & { addCredential(id: string, cred: Credential): void; - waitForResult(attemptId: string): Promise; + addDataclip(id: string, data: DataClip): void; enqueueAttempt(attempt: Attempt): void; - reset(): void; + getAttempt(id: string): Attempt; + getCredential(id: string): Credential; + getDataclip(id: string): DataClip; getQueueLength(): number; getResult(attemptId: string): any; + getState(): ServerState; on(event: LightningEvents, fn: (evt: any) => void): void; once(event: LightningEvents, fn: (evt: any) => void): void; + onSocketEvent( + event: LightningEvents, + attemptId: string, + fn: (evt: any) => void + ): void; + registerAttempt(attempt: Attempt): void; + reset(): void; + startAttempt(id: string): any; + waitForResult(attemptId: string): Promise; }; -const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { +type Api = { + startAttempt(attemptId: string): void; +}; + +const setupDevAPI = ( + app: DevApp, + state: ServerState, + logger: Logger, + api: Api +) => { // Dev APIs for unit testing app.addCredential = (id: string, cred: Credential) => { logger.info(`Add credential ${id}`); @@ -40,9 +63,12 @@ const setupDevAPI = (app: DevApp, state: ServerState, logger: Logger, api) => { app.getDataclip = (id: string) => state.dataclips[id]; - app.enqueueAttempt = (attempt: Attempt) => { + app.enqueueAttempt = (attempt: Attempt, workerId = 'rtm') => { state.attempts[attempt.id] = attempt; - state.results[attempt.id] = {}; + state.results[attempt.id] = { + workerId, // TODO + state: null, + }; state.pending[attempt.id] = { status: 'queued', logs: [], @@ -122,7 +148,7 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { const router = new Router(); router.post('/attempt', (ctx) => { - const attempt = ctx.request.body; + const attempt = ctx.request.body as Attempt; if (!attempt) { ctx.response.status = 400; @@ -154,7 +180,7 @@ const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { return router.routes(); }; -export default (app: DevApp, state: ServerState, logger: Logger, api) => { +export default (app: DevApp, state: ServerState, logger: Logger, api: Api) => { setupDevAPI(app, state, logger, api); return setupRestAPI(app, state, logger); }; diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts index b7c641a21..12e90f475 100644 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ b/packages/ws-worker/src/mock/lightning/api-sockets.ts @@ -265,13 +265,16 @@ const createSocketAPI = ( attemptId: string ) { const { ref, join_ref, topic, payload } = evt; - const { dataclip } = payload; + const { final_dataclip_id } = payload; logger?.info('Completed attempt ', attemptId); - logger?.debug(dataclip); + logger?.debug(final_dataclip_id); state.pending[attemptId].status = 'complete'; - state.results[attemptId] = dataclip; + if (!state.results[attemptId]) { + state.results[attemptId] = { state: null, workerId: 'mock' }; + } + state.results[attemptId].state = state.dataclips[final_dataclip_id]; ws.reply({ ref, @@ -307,14 +310,14 @@ const createSocketAPI = ( ws: DevSocket, evt: PhoenixEvent ) { - const { ref, join_ref, topic, payload } = evt; + const { ref, join_ref, topic } = evt; const { output_dataclip_id, output_dataclip } = evt.payload; if (output_dataclip_id) { if (!state.dataclips) { state.dataclips = {}; } - state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip); + state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip!); } // be polite and acknowledge the event diff --git a/packages/ws-worker/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts index 3d1ac66d7..67f6aa8b3 100644 --- a/packages/ws-worker/src/mock/lightning/server.ts +++ b/packages/ws-worker/src/mock/lightning/server.ts @@ -6,16 +6,16 @@ import createLogger, { createMockLogger, LogLevel, Logger, - JSONLog, } from '@openfn/logger'; import createWebSocketAPI from './api-sockets'; import createDevAPI from './api-dev'; import { Attempt } from '../../types'; +import { ATTEMPT_LOG_PAYLOAD } from '../../events'; export type AttemptState = { status: 'queued' | 'started' | 'complete'; - logs: JSONLog[]; + logs: ATTEMPT_LOG_PAYLOAD[]; }; export type ServerState = { @@ -89,7 +89,7 @@ const createLightningServer = (options: LightningOptions = {}) => { app.use(createDevAPI(app as any, state, logger, api)); - app.destroy = () => { + (app as any).destroy = () => { server.close(); }; diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts index 0fc1f0391..2655a2c9a 100644 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ b/packages/ws-worker/src/mock/lightning/socket-server.ts @@ -6,6 +6,7 @@ */ import { WebSocketServer, WebSocket } from 'ws'; import querystring from 'query-string'; +// @ts-ignore import { Serializer } from 'phoenix'; import { ATTEMPT_PREFIX, extractAttemptId } from './util'; @@ -132,6 +133,7 @@ function createServer({ }, }; + // @ts-ignore something wierd about the wsServer typing wsServer.on('connection', function (ws: DevSocket, req: any) { logger?.info('new client connected'); @@ -166,6 +168,7 @@ function createServer({ topic, payload, }); + // @ts-ignore ws.send(evt); }; @@ -177,6 +180,7 @@ function createServer({ topic, payload: stringify(payload), // TODO do we stringify this? All of it? }); + // @ts-ignore ws.send(evt); }; @@ -191,8 +195,9 @@ function createServer({ logger?.debug(`>> [${topic}] ${event} ${ref} :: ${stringify(payload)}`); - if (events[event]) { + if (event in events) { // handle system/phoenix events + // @ts-ignore events[event](ws, { topic, payload, ref, join_ref }); } else { // handle custom/user events diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 45cacfb83..cbae11f75 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -16,7 +16,7 @@ type Resolver = (id: string) => Promise; // A list of helper functions which basically resolve ids into JSON // to lazy load assets export type LazyResolvers = { - credentials?: Resolver; + credential?: Resolver; state?: Resolver; expressions?: Resolver; }; @@ -101,7 +101,7 @@ function createMock(serverId?: string) { if (typeof configuration === 'string') { // Fetch the credential but do nothing with it // Maybe later we use it to assemble state - await resolvers.credential(configuration); + await resolvers.credential?.(configuration); } const info = (...message: any[]) => { @@ -181,9 +181,6 @@ function createMock(serverId?: string) { once, execute, getStatus, - setResolvers: (r: LazyResolvers) => { - resolvers = r; - }, listen, }; } diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 2d77bdd44..32b885407 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -4,7 +4,6 @@ import koaLogger from 'koa-logger'; import Router from '@koa/router'; import { createMockLogger, Logger } from '@openfn/logger'; -import createRestAPI from './api-rest'; import startWorkloop from './api/workloop'; import claim from './api/claim'; import { execute } from './api/execute'; @@ -39,8 +38,6 @@ function createServer(engine: any, options: ServerOptions = {}) { }) ); - createRestAPI(app, logger); - app.listen(port); logger.success('ws-worker listening on', port); @@ -94,13 +91,12 @@ function createServer(engine: any, options: ServerOptions = {}) { router.post('/claim', async (ctx) => { logger.info('triggering claim from POST request'); return claim(channel, startAttempt, logger) - .then(({ id, token }) => { + .then(() => { logger.info('claim complete: 1 attempt claimed'); - startAttempt({ id, token }); ctx.body = 'complete'; ctx.status = 200; }) - .catch((e) => { + .catch(() => { logger.info('claim complete: no attempts'); ctx.body = 'no attempts'; ctx.status = 204; diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index d2b6cc740..b1176e1c1 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -1,7 +1,7 @@ // start the server in a local CLI import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import createLogger from '@openfn/logger'; +import createLogger, { LogLevel } from '@openfn/logger'; // import createRTM from '@openfn/runtime-engine'; import createMockRTE from './mock/runtime-engine'; @@ -14,6 +14,7 @@ type Args = { repoDir?: string; secret?: string; loop?: boolean; + log: LogLevel; }; const args = yargs(hideBin(process.argv)) diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index 9aa130741..ea63ce151 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -1,3 +1,7 @@ +import type { Socket as PhxSocket, Channel as PhxChannel } from 'phoenix'; + +export { Socket }; + export type Credential = Record; export type State = { @@ -56,20 +60,20 @@ type ReceiveHook = { ) => ReceiveHook; }; -export declare class Socket { - constructor(endpoint: string, options: { params: any }); - onOpen(callback: () => void): void; - connect(): void; - channel(channelName: string, params: any): Channel; -} +// export declare class Socket extends PhxSocket { +// constructor(endpoint: string, options: { params: any }); +// onOpen(callback: () => void): void; +// connect(): void; +// channel(channelName: string, params: any): Channel; +// } -export type Channel = { - on: (event: string, fn: (evt: any) => void) => void; +export interface Channel extends PhxChannel { + // on: (event: string, fn: (evt: any) => void) => void; // TODO it would be super nice to infer the event from the payload - push:

(event: string, payload?: P) => ReceiveHook; - join: () => ReceiveHook; -}; + push:

(event: string, payload?: P) => ReceiveHook; + // join: () => ReceiveHook; +} // type RuntimeExecutionPlanID = string; // type JobEdge = { diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts index 60ed0bda7..706174868 100644 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ b/packages/ws-worker/test/mock/lightning.test.ts @@ -228,23 +228,27 @@ test.serial('get attempt data through the attempt channel', async (t) => { }); }); -test.serial('complete an attempt through the attempt channel', async (t) => { - return new Promise(async (done) => { - const a = attempt1; - server.registerAttempt(a); - server.startAttempt(a.id); - - const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); - channel - .push(ATTEMPT_COMPLETE, { dataclip: { answer: 42 } }) - .receive('ok', () => { - const { pending, results } = server.getState(); - t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); - t.deepEqual(results[a.id], { answer: 42 }); - done(); - }); - }); -}); +test.serial.only( + 'complete an attempt through the attempt channel', + async (t) => { + return new Promise(async (done) => { + const a = attempt1; + server.registerAttempt(a); + server.startAttempt(a.id); + server.addDataclip('abc', { answer: 42 }); + + const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); + channel + .push(ATTEMPT_COMPLETE, { final_dataclip_id: 'abc' }) + .receive('ok', () => { + const { pending, results } = server.getState(); + t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); + t.deepEqual(results[a.id].state, { answer: 42 }); + done(); + }); + }); + } +); test.serial('logs are saved and acknowledged', async (t) => { return new Promise(async (done) => { diff --git a/packages/ws-worker/tsconfig.json b/packages/ws-worker/tsconfig.json index 7d1bac97f..7a3e3bb21 100644 --- a/packages/ws-worker/tsconfig.json +++ b/packages/ws-worker/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.common", - "include": ["src/**/*.ts", "test/mock/data.ts"], + "include": ["src/**/*.ts", "test/mock/data.ts", "src/api/channels/attempts"], "compilerOptions": { "module": "ESNext" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8daa9830e..4ac183c39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -513,12 +513,24 @@ importers: '@types/koa-bodyparser': specifier: ^4.3.10 version: 4.3.10 - '@types/koa-router': - specifier: ^7.4.4 - version: 7.4.4 + '@types/koa-route': + specifier: ^3.2.6 + version: 3.2.6 + '@types/koa-websocket': + specifier: ^5.0.8 + version: 5.0.8 + '@types/koa__router': + specifier: ^12.0.1 + version: 12.0.1 '@types/node': specifier: ^18.15.3 version: 18.15.13 + '@types/nodemon': + specifier: 1.19.3 + version: 1.19.3 + '@types/phoenix': + specifier: ^1.6.2 + version: 1.6.2 '@types/yargs': specifier: ^17.0.12 version: 17.0.24 @@ -1666,10 +1678,19 @@ packages: '@types/koa': 2.13.5 dev: false - /@types/koa-router@7.4.4: - resolution: {integrity: sha512-3dHlZ6CkhgcWeF6wafEUvyyqjWYfKmev3vy1PtOmr0mBc3wpXPU5E8fBBd4YQo5bRpHPfmwC5yDaX7s4jhIN6A==} + /@types/koa-route@3.2.6: + resolution: {integrity: sha512-g7alI5btLD5AjoWsZeLmcZe0Ey9ozxHqpG6T+N8+bHvp/QF+nKM5iHeIMMWxSk9AUlGeH6fO9p6K/3C7h6Cd8A==} dependencies: '@types/koa': 2.13.5 + path-to-regexp: 1.8.0 + dev: true + + /@types/koa-websocket@5.0.8: + resolution: {integrity: sha512-KHHsgUrdBn4utRFZqlsVPOC0HqxWrnt4rVvuxw0NLb15e8KmVC9xtRru1A4NsquLNJg3hXddWsEf4+x5+NrJXg==} + dependencies: + '@types/koa': 2.13.5 + '@types/koa-compose': 3.2.5 + '@types/ws': 8.5.6 dev: true /@types/koa@2.13.5: @@ -1684,6 +1705,12 @@ packages: '@types/koa-compose': 3.2.5 '@types/node': 18.15.13 + /@types/koa__router@12.0.1: + resolution: {integrity: sha512-uqV+v6pCsfLZwK+Ar6XavKSZ6Cbsgw12bCEX9L0IKHj81LTWXcrayxJWkLtez5vOMQlq+ax+lZcuCyh9CgxYGw==} + dependencies: + '@types/koa': 2.13.5 + dev: true + /@types/live-server@1.2.1: resolution: {integrity: sha512-Yind497JdcZT8L9FF7u73nq44KmamiDitsZJEwrAi/pgBhFHThNvtR+2Z/YGNSMjyUoDBFdvhVSQmod06yd1Ng==} dev: true @@ -1736,10 +1763,20 @@ packages: '@types/node': 18.15.13 dev: true + /@types/nodemon@1.19.3: + resolution: {integrity: sha512-LcKdWgch8uHOF73yYpdE7YPVLT0HnFI60zyNBpJyfAiDDwPy3WAxReQeB84UseE8e8qdJsBqmFXWbjxv7jlXBg==} + dependencies: + '@types/node': 18.15.13 + dev: true + /@types/normalize-package-data@2.4.1: resolution: {integrity: sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==} dev: true + /@types/phoenix@1.6.2: + resolution: {integrity: sha512-I3mm7x5XIi+5NsIY3nfreY+H4PmQdyBwJ84SiUSOxSg1axwEPNmkKWYVm56y+emDpPPUL3cPzrLcgRWSd9gI7g==} + dev: true + /@types/prop-types@15.7.5: resolution: {integrity: sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==} dev: true @@ -1821,7 +1858,6 @@ packages: resolution: {integrity: sha512-8B5EO9jLVCy+B58PLHvLDuOD8DRVMgQzq8d55SjLCOn9kqGyqOvy27exVaTio1q1nX5zLu8/6N0n2ThSxOM6tg==} dependencies: '@types/node': 18.15.13 - dev: false /@types/yargs-parser@21.0.0: resolution: {integrity: sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==} From d4c744f67f9a7a0f1ceb407c7a57224fba6af8c0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 5 Oct 2023 18:05:21 +0100 Subject: [PATCH 128/232] engine: disable type checking --- packages/engine-multi/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 70730caca..267359bd1 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -7,7 +7,7 @@ "private": true, "scripts": { "test": "pnpm ava", - "test:types": "pnpm tsc --noEmit --project tsconfig.json", + "_test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", "build:watch": "pnpm build --watch" }, From 5be95d4acc52ec4752bf0ea2515793eb5f3e63bc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 6 Oct 2023 15:39:56 +0100 Subject: [PATCH 129/232] engine: drop in some typings --- packages/engine-multi/src/types.d.ts | 101 ++++++++++++++++++ .../test/{engine.ts => engine.test.ts} | 2 +- packages/ws-worker/src/mock/runtime-engine.ts | 2 +- 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 packages/engine-multi/src/types.d.ts rename packages/engine-multi/test/{engine.ts => engine.test.ts} (98%) diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts new file mode 100644 index 000000000..7086ec3bf --- /dev/null +++ b/packages/engine-multi/src/types.d.ts @@ -0,0 +1,101 @@ +// ok first of allI want to capture the key interfaces + +import { ExecutionPlan } from '@openfn/runtime'; + +type WorkflowStartEvent = 'workflow-start'; +type WorkflowStartPayload = { + workflowId: string; +}; + +type WorkflowCompleteEvent = 'workflow-complete'; +type WorkflowCompletePayload = { + workflowId: string; +}; + +// This occurs when a critical error causes the workflow to be aborted +// ie crash, syntax error +type WorkflowErrorEvent = 'workflow-error'; +type WorkflowErrorPayload = { + workflowId: string; + type: string; + message: string; +}; + +type JobStartEvent = 'job-start'; +type JobStartPayload = { + workflowId: string; + jobId: string; +}; + +type JobCompleteEvent = 'job-complete'; +type JobCompletePayload = { + workflowId: string; + jobId: string; + state: any; // the result start +}; + +// a log message coming out of the engine, adaptor or job +type LogEvent = 'job-complete'; +type LogPayload = JSONLog & { + workflowId: string; +}; + +// TODO +type EdgeResolvedEvent = 'edge-resolved'; +type EdgeResolvedPayload = { + workflowId: string; + edgeId: string; // interesting, we don't really have this yet. Is index more appropriate? key? yeah, it's target node basically + result: boolean; +}; + +type EngineEvents = + | WorkflowStartEvent + | WorkflowCompleteEvent + | JobStartEvent + | JobCompleteEvent + | LogEvent; + +type EventPayloadLookup = { + [WorkflowStartEvent]: WorkflowStartPayload; + [WorkflowCompleteEvent]: WorkflowCompletePayload; + [JobStartEvent]: JobStartPayload; + [JobCompleteEvent]: JobCompletePayload; + [LogEvent]: LogPayload; +}; + +type EventHandler = ( + event: EventPayloadLookup[T] +) => void; + +type Resolver = (id: string) => Promise; + +type Resolvers = { + credential?: Resolver; + state?: Resolver; +}; + +type ExecuteOptions = { + sanitize: any; // log sanitise options + noCompile: any; // skip compilation (useful in test) +}; + +interface RuntimeEngine extends EventEmitter { + //id: string // human readable instance id + // actually I think the id is on the worker, not the engine + + // TODO should return an unsubscribe hook + listen( + attemptId: string, + listeners: Record + ): void; + + // TODO return a promise? + // Kinda convenient but not actually needed + execute( + plan: ExecutionPlan, + resolvers: Resolvers, + options: ExecuteOptions = {} + ); + + // TODO my want some maintenance APIs, like getSatus. idk +} diff --git a/packages/engine-multi/test/engine.ts b/packages/engine-multi/test/engine.test.ts similarity index 98% rename from packages/engine-multi/test/engine.ts rename to packages/engine-multi/test/engine.test.ts index 61fbdb716..9c0f86d84 100644 --- a/packages/engine-multi/test/engine.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -3,7 +3,7 @@ import path from 'node:path'; import { createMockLogger } from '@openfn/logger'; import { createPlan } from './util'; -import Manager from '../src/rtm'; +import Manager from '../src/engine'; import * as e from '../src/events'; const logger = createMockLogger('', { level: 'debug' }); diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index cbae11f75..7b6a470cd 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -138,7 +138,7 @@ function createMock(serverId?: string) { // Start executing an ExecutionPlan // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity - const execute = async ( + const execute = ( xplan: ExecutionPlan, resolvers: LazyResolvers = mockResolvers ) => { From 07446be361b9ef245918ef37cb4477fc06fc4260 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 6 Oct 2023 16:16:50 +0100 Subject: [PATCH 130/232] engine: refactor the runners to not be factories This is way easeier for testing and understanding --- packages/engine-multi/src/engine.ts | 37 +++-- .../engine-multi/src/runners/autoinstall.ts | 132 +++++++++--------- packages/engine-multi/src/runners/compile.ts | 41 +++--- packages/engine-multi/src/runners/execute.ts | 51 +++---- .../test/runners/autoinstall.test.ts | 67 +++++---- .../engine-multi/test/runners/compile.test.ts | 8 ++ 6 files changed, 178 insertions(+), 158 deletions(-) create mode 100644 packages/engine-multi/test/runners/compile.test.ts diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index bc597f876..106bbbb24 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -4,13 +4,12 @@ import { fileURLToPath } from 'url'; import { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; import { ExecutionPlan } from '@openfn/runtime'; +import createLogger, { JSONLog, Logger } from '@openfn/logger'; import * as e from './events'; -// import createAutoinstall from './runners/autoinstall'; -import createCompile from './runners/compile'; -import createExecute from './runners/execute'; -import createLogger, { JSONLog, Logger } from '@openfn/logger'; -import createAutoInstall from './runners/autoinstall'; +import compile from './runners/compile'; +import execute from './runners/execute'; +import autoinstall from './runners/autoinstall'; export type State = any; // TODO I want a nice state def with generics @@ -146,20 +145,16 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { }); }; - // Create "runner" functions for execute and compile - const execute = createExecute(workers, logger, { - start: onWorkflowStarted, - log: onWorkflowLog, - }); - const compile = createCompile(logger, repoDir); - - const autoinstall = createAutoInstall({ repoDir, logger }); - // How much of this happens inside the worker? // Shoud the main thread handle compilation? Has to if we want to cache // Unless we create a dedicated compiler worker // TODO error handling, timeout const handleExecute = async (plan: ExecutionPlan) => { + const options = { + repoDir, + }; + const context = { plan, logger, workers, options /* api */ }; + logger.debug('Executing workflow ', plan.id); allWorkflows.set(plan.id!, { @@ -168,14 +163,18 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { plan, }); - const adaptorPaths = await autoinstall(plan); + const adaptorPaths = await autoinstall(context); - // Don't compile if we're running a mock (not a fan of this) - const compiledPlan = noCompile ? plan : await compile(plan); + if (!noCompile) { + context.plan = await compile(context); + } logger.debug('workflow compiled ', plan.id); - const result = await execute(compiledPlan, adaptorPaths); - completeWorkflow(compiledPlan.id!, result); + const result = await execute(context, adaptorPaths, { + start: onWorkflowStarted, + log: onWorkflowLog, + }); + completeWorkflow(plan.id!, result); logger.debug('finished executing workflow ', plan.id); // Return the result diff --git a/packages/engine-multi/src/runners/autoinstall.ts b/packages/engine-multi/src/runners/autoinstall.ts index 96084e14d..7f8587270 100644 --- a/packages/engine-multi/src/runners/autoinstall.ts +++ b/packages/engine-multi/src/runners/autoinstall.ts @@ -7,17 +7,80 @@ import { getNameAndVersion, loadRepoPkg, } from '@openfn/runtime'; -import { install } from '@openfn/runtime'; +import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; +type Options = { + repoDir: string; + logger: Logger; + skipRepoValidation?: boolean; + handleInstall?( + fn: string, + options?: Pick + ): Promise; + handleIsInstalled?( + fn: string, + options?: Pick + ): Promise; +}; + +const pending: Record> = {}; + +const autoinstall = async (context: any): Promise => { + const { options, plan, logger } = context; + const { repoDir } = options; + + const installFn = options.handleInstall || install; + const isInstalledFn = options.handleIsInstalled || isInstalled; + + let didValidateRepo = false; + const { skipRepoValidation } = options; + + if (!skipRepoValidation && !didValidateRepo && options.repoDir) { + // TODO what if this throws? + // Whole server probably needs to crash, so throwing is probably appropriate + await ensureRepo(options.repoDir, options.logger); + didValidateRepo = true; + } + + const adaptors = Array.from(identifyAdaptors(plan)); + // TODO would rather do all this in parallel but this is fine for now + // TODO set iteration is weirdly difficult? + const paths: ModulePaths = {}; + for (const a of adaptors) { + // Return a path name to this module for the linker to use later + // TODO this is all a bit rushed + const alias = getAliasedName(a); + const { name } = getNameAndVersion(a); + paths[name] = { path: `${repoDir}/node_modules/${alias}` }; + + const needsInstalling = !(await isInstalledFn(a, options.repoDir, logger)); + if (needsInstalling) { + if (!pending[a]) { + // add a promise to the pending array + pending[a] = installFn(a, options.repoDir, logger); + } + // Return the pending promise (safe to do this multiple times) + await pending[a].then(); + } + } + return paths; +}; + +export default autoinstall; + // The actual install function is not unit tested // It's basically just a proxy to @openfn/runtime -const doHandleInstall = (specifier: string, options: Options) => - install(specifier, options.repoDir, options.logger); +const install = (specifier: string, repoDir: string, logger: Logger) => + runtimeInstall(specifier, repoDir, logger); // The actual isInstalled function is not unit tested // TODO this should probably all be handled (and tested) in @openfn/runtime -const doIsInstalled = async (specifier: string, options: Options) => { +const isInstalled = async ( + specifier: string, + repoDir: string, + logger: Logger +) => { const alias = getAliasedName(specifier); if (!alias.match('_')) { // Note that if the adaptor has no version number, the alias will be "wrong" @@ -25,12 +88,12 @@ const doIsInstalled = async (specifier: string, options: Options) => { // The install function will later decide a version number and may, or may // not, install for us. // This log isn't terrible helpful as there's no attempt version info - options.logger.warn( + logger.warn( `adaptor ${specifier} does not have a version number - will attempt to auto-install` ); } // TODO is it really appropriate to load this file each time? - const pkg = await loadRepoPkg(options.repoDir); + const pkg = await loadRepoPkg(repoDir); if (pkg) { const { dependencies } = pkg; return dependencies.hasOwnProperty(alias); @@ -45,61 +108,4 @@ export const identifyAdaptors = (plan: ExecutionPlan): Set => { return adaptors; }; -type Options = { - repoDir: string; - logger: Logger; - skipRepoValidation?: boolean; - handleInstall?( - fn: string, - options?: Pick - ): Promise; - handleIsInstalled?( - fn: string, - options?: Pick - ): Promise; -}; - export type ModulePaths = Record; - -const createAutoInstall = (options: Options) => { - const install = options.handleInstall || doHandleInstall; - const isInstalled = options.handleIsInstalled || doIsInstalled; - const pending: Record> = {}; - - let didValidateRepo = false; - const { skipRepoValidation } = options; - - return async (plan: ExecutionPlan): Promise => { - if (!skipRepoValidation && !didValidateRepo && options.repoDir) { - // TODO what if this throws? - // Whole server probably needs to crash, so throwing is probably appropriate - await ensureRepo(options.repoDir, options.logger); - didValidateRepo = true; - } - - const adaptors = Array.from(identifyAdaptors(plan)); - // TODO would rather do all this in parallel but this is fine for now - // TODO set iteration is weirdly difficult? - const paths: ModulePaths = {}; - for (const a of adaptors) { - // Return a path name to this module for the linker to use later - // TODO this is all a bit rushed - const alias = getAliasedName(a); - const { name } = getNameAndVersion(a); - paths[name] = { path: `${options.repoDir}/node_modules/${alias}` }; - - const needsInstalling = !(await isInstalled(a, options)); - if (needsInstalling) { - if (!pending[a]) { - // add a promise to the pending array - pending[a] = install(a, options); - } - // Return the pending promise (safe to do this multiple times) - await pending[a].then(); - } - } - return paths; - }; -}; - -export default createAutoInstall; diff --git a/packages/engine-multi/src/runners/compile.ts b/packages/engine-multi/src/runners/compile.ts index 11089c9c7..366493e4d 100644 --- a/packages/engine-multi/src/runners/compile.ts +++ b/packages/engine-multi/src/runners/compile.ts @@ -3,31 +3,32 @@ // being compiled twice import type { Logger } from '@openfn/logger'; -import compile, { preloadAdaptorExports, Options } from '@openfn/compiler'; +import compile, { preloadAdaptorExports } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; -const createCompile = (logger: Logger, repoDir: string) => { - const cache = {}; - return async (plan) => { - // Compile each job in the exeuction plan - // A bit like the CLI - for (const job of plan.jobs) { - if (job.expression) { - job.expression = await compileJob( - job.expression as string, - job.adaptor, // TODO need to expand this. Or do I? - repoDir, - logger - ); - } +// TODO this compiler is going to change anyway to run just in time +// the runtime will have an onCompile hook +// We'll keep this for now though while we get everything else working +export default async (context: any) => { + const { logger, options, plan } = context; + const { repoDir } = options; + + for (const job of plan.jobs) { + if (job.expression) { + job.expression = await compileJob( + job.expression as string, + job.adaptor, // TODO need to expand this. Or do I? + repoDir, + logger + ); } - return plan; - }; + } + return plan; }; -export default createCompile; - -// TODO copied out of CLI +// TODO copied out of CLI - how can we share code here? +// engine should not have a dependency on the cli +// maybe this is a runtime util const stripVersionSpecifier = (specifier: string) => { const idx = specifier.lastIndexOf('@'); if (idx > 0) { diff --git a/packages/engine-multi/src/runners/execute.ts b/packages/engine-multi/src/runners/execute.ts index 69c1fb24d..783104dc7 100644 --- a/packages/engine-multi/src/runners/execute.ts +++ b/packages/engine-multi/src/runners/execute.ts @@ -1,8 +1,4 @@ // Execute a compiled workflow -import type { WorkerPool } from 'workerpool'; -import type { ExecutionPlan } from '@openfn/runtime'; -import { Logger } from '@openfn/logger'; - import * as e from '../events'; import { ModulePaths } from './autoinstall'; @@ -10,30 +6,29 @@ import { ModulePaths } from './autoinstall'; // Is it better to just return the handler? // But then this function really isn't doing so much // (I guess that's true anyway) -const execute = (workers: WorkerPool, logger: Logger, events: any) => { - const { start, log, error } = events; - return (plan: ExecutionPlan, adaptorPaths: ModulePaths) => { - return new Promise((resolve) => - workers - .exec('run', [plan, adaptorPaths], { - on: ({ type, ...args }: e.WorkflowEvent) => { - if (type === e.WORKFLOW_START) { - const { workflowId, threadId } = args as e.AcceptWorkflowEvent; - start?.(workflowId, threadId); - } else if (type === e.WORKFLOW_COMPLETE) { - const { workflowId, state } = args as e.CompleteWorkflowEvent; - resolve(state); - } else if (type === e.WORKFLOW_LOG) { - const { workflowId, message } = args as e.LogWorkflowEvent; - log(workflowId, message); - } - }, - }) - .catch((e) => { - logger.error(e); - }) - ); - }; +const execute = (context: any, adaptorPaths: ModulePaths, events: any) => { + const { workers, logger, plan } = context; + const { start, log } = events; + return new Promise((resolve) => + workers + .exec('run', [plan, adaptorPaths], { + on: ({ type, ...args }: e.WorkflowEvent) => { + if (type === e.WORKFLOW_START) { + const { workflowId, threadId } = args as e.AcceptWorkflowEvent; + start?.(workflowId, threadId); + } else if (type === e.WORKFLOW_COMPLETE) { + const { state } = args as e.CompleteWorkflowEvent; + resolve(state); + } else if (type === e.WORKFLOW_LOG) { + const { workflowId, message } = args as e.LogWorkflowEvent; + log(workflowId, message); + } + }, + }) + .catch((e: any) => { + logger.error(e); + }) + ); }; export default execute; diff --git a/packages/engine-multi/test/runners/autoinstall.test.ts b/packages/engine-multi/test/runners/autoinstall.test.ts index d32bc547b..314d41750 100644 --- a/packages/engine-multi/test/runners/autoinstall.test.ts +++ b/packages/engine-multi/test/runners/autoinstall.test.ts @@ -1,7 +1,5 @@ import test from 'ava'; -import createAutoInstall, { - identifyAdaptors, -} from '../../src/runners/autoinstall'; +import autoinstall, { identifyAdaptors } from '../../src/runners/autoinstall'; const mockIsInstalled = (pkg) => async (specifier: string) => { const alias = specifier.split('@').join('_'); @@ -63,7 +61,7 @@ test('identifyAdaptors: pick out adaptors and remove duplicates', (t) => { }); // This doesn't do anything except check that the mocks are installed -test('autoinstall: should call both mock functions', (t) => { +test('autoinstall: should call both mock functions', async (t) => { let didCallIsInstalled = false; let didCallInstall = true; @@ -76,19 +74,21 @@ test('autoinstall: should call both mock functions', (t) => { return; }; - const autoinstall = createAutoInstall({ - handleInstall: mockInstall, - handleIsInstalled: mockIsInstalled, - }); - - autoinstall({ - jobs: [{ adaptor: 'x@1.0.0' }], + await autoinstall({ + options: { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }, + plan: { + jobs: [{ adaptor: 'x@1.0.0' }], + }, }); t.true(didCallIsInstalled); t.true(didCallInstall); }); +// TODO a problem with this test is that pending state is shared across tests test('autoinstall: only call install once if there are two concurrent install requests', async (t) => { let callCount = 0; @@ -98,18 +98,27 @@ test('autoinstall: only call install once if there are two concurrent install re setTimeout(() => resolve(), 20); }); - const autoinstall = createAutoInstall({ - handleInstall: mockInstall, - handleIsInstalled: async () => false, - }); - - autoinstall({ - jobs: [{ adaptor: 'x@1.0.0' }], - }); + await Promise.all([ + autoinstall({ + options: { + handleInstall: mockInstall, + handleIsInstalled: async () => false, + }, + plan: { + jobs: [{ adaptor: 'z@1.0.0' }], + }, + }), - await autoinstall({ - jobs: [{ adaptor: 'x@1.0.0' }], - }); + autoinstall({ + options: { + handleInstall: mockInstall, + handleIsInstalled: async () => false, + }, + plan: { + jobs: [{ adaptor: 'z@1.0.0' }], + }, + }), + ]); t.is(callCount, 1); }); @@ -127,14 +136,16 @@ test('autoinstall: return a map to modules', async (t) => { ], }; - const autoinstall = createAutoInstall({ - repoDir: 'a/b/c', - skipRepoValidation: true, - handleInstall: async () => true, - handleIsInstalled: async () => false, + const result = await autoinstall({ + plan, + options: { + repoDir: 'a/b/c', + skipRepoValidation: true, + handleInstall: async () => true, + handleIsInstalled: async () => false, + }, }); - const result = await autoinstall(plan); t.deepEqual(result, { common: { path: 'a/b/c/node_modules/common_1.0.0' }, http: { path: 'a/b/c/node_modules/http_1.0.0' }, diff --git a/packages/engine-multi/test/runners/compile.test.ts b/packages/engine-multi/test/runners/compile.test.ts new file mode 100644 index 000000000..214a689ad --- /dev/null +++ b/packages/engine-multi/test/runners/compile.test.ts @@ -0,0 +1,8 @@ +import test from 'ava'; + +// Because compile is going to change I'm not too excited about doing a lot of work here right nopw +// just filling the space + +test.todo('should compile multiple expressions in a job'); + +test.todo('should log as it compiles'); From be11be67b30e051ab24727fce948ef111e2d3fe7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 10 Oct 2023 17:07:06 +0100 Subject: [PATCH 131/232] engine: part way through huge refactor --- packages/engine-multi/src/api.ts | 119 +++++++++++++ .../src/{runners => api}/autoinstall.ts | 26 +-- packages/engine-multi/src/api/call-worker.ts | 38 +++++ .../src/{runners => api}/compile.ts | 10 +- packages/engine-multi/src/api/create-state.ts | 17 ++ packages/engine-multi/src/api/execute.ts | 37 ++++ packages/engine-multi/src/api/lifecycle.ts | 98 +++++++++++ packages/engine-multi/src/engine.ts | 161 ++++++++++-------- packages/engine-multi/src/events.ts | 14 +- packages/engine-multi/src/runners/execute.ts | 34 ---- packages/engine-multi/src/types.d.ts | 50 +++++- packages/engine-multi/src/worker-helper.ts | 4 +- packages/engine-multi/src/worker.ts | 1 + packages/engine-multi/test/api.test.ts | 20 +++ .../test/{runners => api}/autoinstall.test.ts | 130 +++++++------- .../engine-multi/test/api/call-worker.test.ts | 64 +++++++ .../test/{runners => api}/compile.test.ts | 0 .../engine-multi/test/api/lifecycle.test.ts | 124 ++++++++++++++ .../engine-multi/test/worker-functions.js | 16 ++ 19 files changed, 776 insertions(+), 187 deletions(-) create mode 100644 packages/engine-multi/src/api.ts rename packages/engine-multi/src/{runners => api}/autoinstall.ts (85%) create mode 100644 packages/engine-multi/src/api/call-worker.ts rename packages/engine-multi/src/{runners => api}/compile.ts (88%) create mode 100644 packages/engine-multi/src/api/create-state.ts create mode 100644 packages/engine-multi/src/api/execute.ts create mode 100644 packages/engine-multi/src/api/lifecycle.ts delete mode 100644 packages/engine-multi/src/runners/execute.ts create mode 100644 packages/engine-multi/test/api.test.ts rename packages/engine-multi/test/{runners => api}/autoinstall.test.ts (54%) create mode 100644 packages/engine-multi/test/api/call-worker.test.ts rename packages/engine-multi/test/{runners => api}/compile.test.ts (100%) create mode 100644 packages/engine-multi/test/api/lifecycle.test.ts create mode 100644 packages/engine-multi/test/worker-functions.js diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts new file mode 100644 index 000000000..243348f7f --- /dev/null +++ b/packages/engine-multi/src/api.ts @@ -0,0 +1,119 @@ +import createLogger, { JSONLog, Logger } from '@openfn/logger'; +import { EngineAPI, EngineEvents, EventHandler } from './types'; +import { EventEmitter } from 'node:events'; +import { WORKFLOW_COMPLETE, WORKFLOW_START } from './events'; + +import initWorkers from './api/call-worker'; + +// For each workflow, create an API object with its own event emitter +// this is a bt wierd - what if the emitter went on state instead? +const createWorkflowEvents = (api: EngineAPI) => { + //create a bespoke event emitter + const events = new EventEmitter(); + + // TODO need this in closure really + // listeners[workflowId] = events; + + // proxy all events to the main emitter + // uh actually there may be no point in this + function proxy(event: string) { + events.on(event, (evt) => { + // ensure the attempt id is on the event + evt.workflowId = workflowId; + const newEvt = { + ...evt, + workflowId: workflowId, + }; + + api.emit(event, newEvt); + }); + } + proxy(WORKFLOW_START); + proxy(WORKFLOW_COMPLETE); + proxy(JOB_START); + proxy(JOB_COMPLETE); + proxy(LOG); + + return events; +}; + +const createAPI = (repoDir: string, options) => { + const listeners = {}; + + // but this is more like an internal api, right? + // maybe this is like the workflow context + const state = { + workflows: {}, + }; // TODO maybe this sits in another file + // the state has apis for getting/setting workflow state + // maybe actually each execute setsup its own state object + + // TODO I think there's an internal and external API + // api is the external thing that other people call + // engine is the internal thing + const api = new EventEmitter() as EngineAPI; + + api.logger = options.logger || createLogger('RTE', { level: 'debug' }); + + api.registerWorkflow = (state) => { + state.workflows[plan.id] = state; + }; + + // what if this returns a bespoke event listener? + // i don't need to to execute(); listen, i can just excute + // it's kinda slick but you still need two lines of code and it doesn't buy anyone anything + // also this is nevver gonna get used externally so it doesn't need to be slick + api.execute = (executionPlan) => { + const workflowId = plan.id; + + // Pull options out of the plan so that all options are in one place + const { options, ...plan } = executionPlan; + + // initial state for this workflow run + // TODO let's create a util function for this (nice for testing) + const state = createState(plan); + // the engine does need to be able to report on the state of each workflow + api.registerWorkflow(state); + + const events = createWorkflowEvents(api); + + listeners[workflowId] = events; + + // this context API thing is the internal api / engine + // each has a bespoke event emitter but otherwise a common interface + const contextAPI: EngineAPI = { + ...api, + ...events, + }; + + execute(contextAPI, state, options); + + // return the event emitter (not the full engine API though) + return events; + }; + + // // how will this actually work? + // api.listen = ( + // attemptId: string, + // listeners: Record + // ) => { + // // const handler = (eventName) => { + // // if () + // // } + // const events = listeners[workflowId]; + // for (const evt of listeners) { + // events.on(evt, listeners[evt]); + // } + + // // TODO return unsubscribe handle + // }; + + // we can have global reporting like this + api.getStatus = (workflowId) => state.workflows[workflowId].status; + + initWorkers(api); + + return api; +}; + +export default createAPI; diff --git a/packages/engine-multi/src/runners/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts similarity index 85% rename from packages/engine-multi/src/runners/autoinstall.ts rename to packages/engine-multi/src/api/autoinstall.ts index 7f8587270..5f52db05c 100644 --- a/packages/engine-multi/src/runners/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -9,25 +9,29 @@ import { } from '@openfn/runtime'; import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; +import { EngineAPI, WorkflowState } from '../types'; +// none of these options should be on the plan actually type Options = { - repoDir: string; - logger: Logger; + repoDir?: string; skipRepoValidation?: boolean; - handleInstall?( - fn: string, - options?: Pick - ): Promise; + handleInstall?(fn: string, repoDir: string, logger: Logger): Promise; handleIsInstalled?( fn: string, - options?: Pick + repoDir: string, + logger: Logger ): Promise; }; const pending: Record> = {}; -const autoinstall = async (context: any): Promise => { - const { options, plan, logger } = context; +const autoinstall = async ( + api: EngineAPI, + state: WorkflowState, + options: Options +): Promise => { + const { logger } = api; + const { plan } = state; const { repoDir } = options; const installFn = options.handleInstall || install; @@ -39,7 +43,7 @@ const autoinstall = async (context: any): Promise => { if (!skipRepoValidation && !didValidateRepo && options.repoDir) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate - await ensureRepo(options.repoDir, options.logger); + await ensureRepo(options.repoDir, logger); didValidateRepo = true; } @@ -54,7 +58,7 @@ const autoinstall = async (context: any): Promise => { const { name } = getNameAndVersion(a); paths[name] = { path: `${repoDir}/node_modules/${alias}` }; - const needsInstalling = !(await isInstalledFn(a, options.repoDir, logger)); + const needsInstalling = !(await isInstalledFn(a, repoDir, logger)); if (needsInstalling) { if (!pending[a]) { // add a promise to the pending array diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts new file mode 100644 index 000000000..c62aa97e0 --- /dev/null +++ b/packages/engine-multi/src/api/call-worker.ts @@ -0,0 +1,38 @@ +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import workerpool from 'workerpool'; +import { EngineAPI } from '../types'; + +// All events coming out of the worker need to include a type key +type WorkerEvent = { + type: string; + [key: string]: any; +}; + +// Adds a `callWorker` function to the API object, which will execute a task in a worker +export default function initWorkers(api: EngineAPI, workerPath: string) { + const workers = createWorkers(workerPath); + + api.callWorker = (task: string, args: any[] = [], events: any = {}) => + workers.exec(task, args, { + on: ({ type, ...args }: WorkerEvent) => { + // just call the callback + events[type]?.(args); + }, + }); +} + +export function createWorkers(workerPath: string) { + let resolvedWorkerPath; + if (workerPath) { + // If a path to the worker has been passed in, just use it verbatim + // We use this to pass a mock worker for testing purposes + resolvedWorkerPath = workerPath; + } else { + // By default, we load ./worker.js but can't rely on the working dir to find it + const dirname = path.dirname(fileURLToPath(import.meta.url)); + resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + } + + return workerpool.pool(resolvedWorkerPath); +} diff --git a/packages/engine-multi/src/runners/compile.ts b/packages/engine-multi/src/api/compile.ts similarity index 88% rename from packages/engine-multi/src/runners/compile.ts rename to packages/engine-multi/src/api/compile.ts index 366493e4d..ba58c93eb 100644 --- a/packages/engine-multi/src/runners/compile.ts +++ b/packages/engine-multi/src/api/compile.ts @@ -5,12 +5,18 @@ import type { Logger } from '@openfn/logger'; import compile, { preloadAdaptorExports } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; +import { EngineAPI, WorkflowState } from '../types'; // TODO this compiler is going to change anyway to run just in time // the runtime will have an onCompile hook // We'll keep this for now though while we get everything else working -export default async (context: any) => { - const { logger, options, plan } = context; +export default async ( + api: EngineAPI, + state: WorkflowState, + options: { repoDir: string } +) => { + const { logger } = api; + const { plan } = state; const { repoDir } = options; for (const job of plan.jobs) { diff --git a/packages/engine-multi/src/api/create-state.ts b/packages/engine-multi/src/api/create-state.ts new file mode 100644 index 000000000..d25b07776 --- /dev/null +++ b/packages/engine-multi/src/api/create-state.ts @@ -0,0 +1,17 @@ +export default (plan) => ({ + id: plan.id, + status: 'pending', + plan, + + threadId: undefined, + startTime: undefined, + duration: undefined, + error: undefined, + result: undefined, + + // yeah not sure about options right now + // options: { + // ...options, + // repoDir, + // }, +}); diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts new file mode 100644 index 000000000..ea89feb1f --- /dev/null +++ b/packages/engine-multi/src/api/execute.ts @@ -0,0 +1,37 @@ +// this replaces the runner + +// Execute a compiled workflow +import * as e from '../events'; +import { EngineAPI, WorkflowState } from '../types'; + +import autoinstall from './autoinstall'; +import compile from './compile'; +import { workflowStart, workflowComplete, log } from './lifecycle'; + +// A lot of callbacks needed here +// Is it better to just return the handler? +// But then this function really isn't doing so much +// (I guess that's true anyway) +const execute = async (api: EngineAPI, state: WorkflowState, options) => { + const adaptorPaths = await autoinstall(api, state, options); + await compile(api, state, options); + + const events = { + [e.WORKFLOW_START]: (evt) => { + workflowStart(api, state, evt); + }, + [e.WORKFLOW_COMPLETE]: (evt) => { + workflowComplete(api, state, evt); + }, + [e.WORKFLOW_LOG]: (evt) => { + log(api, state, evt); + }, + }; + + return api.callWorker('run', [plan, adaptorPaths], events).catch((e) => { + // TODO what about errors then? + api.logger.error(e); + }); +}; + +export default execute; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts new file mode 100644 index 000000000..d84bdffbb --- /dev/null +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -0,0 +1,98 @@ +import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from '../events'; +import { + EngineAPI, + WorkerCompletePayload, + WorkerLogPayload, + WorkerStartPayload, + WorkflowState, +} from '../types'; + +export const workflowStart = ( + api: EngineAPI, // general API + state: WorkflowState, // mutable workflow state + event: WorkerStartPayload // the event published by the runtime itself ({ workflowId, threadId }) +) => { + const { workflowId, threadId } = event; + + api.logger.info('starting workflow ', workflowId); + + // where would this throw get caught? + if (state.startTime) { + // TODO this shouldn't throw.. but what do we do? + // We shouldn't run a workflow that's been run + // Every workflow should have a unique id + // maybe the RTM doesn't care about this + throw new Error(`Workflow with id ${workflowId} is already started`); + } + + Object.assign(state, { + status: 'running', + startTime: Date.now(), + duration: -1, + threadId: threadId, + }); + + // TODO do we still want to push this into the active workflows array? + // api.activeWorkflows.push(workflowId); + + // forward the event on to any external listeners + api.emit(WORKFLOW_START, { + workflowId, // if this is a bespoke emitter it can be implied, which is nice + // Should we publish anything else here? + }); +}; + +export const workflowComplete = ( + api: EngineAPI, // general API + state: WorkflowState, // mutable workflow state + event: WorkerCompletePayload // the event published by the runtime itself ({ workflowId, threadId }) +) => { + const { workflowId, state: result } = event; + + api.logger.success('complete workflow ', workflowId); + api.logger.info(state); + + // TODO I don't know how we'd get here in this architecture + // if (!allWorkflows.has(workflowId)) { + // throw new Error(`Workflow with id ${workflowId} is not defined`); + // } + + state.status = 'done'; + state.result = result; + state.duration = Date.now() - state.startTime!; + + // TODO do we have to remove this from the active workflows array? + // const idx = activeWorkflows.findIndex((id) => id === workflowId); + // activeWorkflows.splice(idx, 1); + + // forward the event on to any external listeners + api.emit(WORKFLOW_COMPLETE, { + id: workflowId, + duration: state.duration, + state: result, + }); +}; + +export const log = ( + api: EngineAPI, // general API + state: WorkflowState, // mutable workflow state + event: WorkerLogPayload // the event published by the runtime itself ({ workflowId, threadId }) +) => { + const { id } = state; + // // TODO not sure about this stuff, I think we can drop it? + // const newMessage = { + // ...message, + // // Prefix the job id in all local jobs + // // I'm sure there are nicer, more elegant ways of doing this + // message: [`[${workflowId}]`, ...message.message], + // }; + api.logger.proxy(event); + + api.emit(WORKFLOW_LOG, { + workflowId: id, + ...event, + }); +}; + +// TODO jobstart +// TODO jobcomplete diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 106bbbb24..f3392ecef 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -7,9 +7,9 @@ import { ExecutionPlan } from '@openfn/runtime'; import createLogger, { JSONLog, Logger } from '@openfn/logger'; import * as e from './events'; -import compile from './runners/compile'; +import compile from './api/compile'; import execute from './runners/execute'; -import autoinstall from './runners/autoinstall'; +import autoinstall from './api/autoinstall'; export type State = any; // TODO I want a nice state def with generics @@ -55,17 +55,28 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { const allWorkflows: Map = new Map(); const activeWorkflows: string[] = []; - let resolvedWorkerPath; - if (workerPath) { - // If a path to the worker has been passed in, just use it verbatim - // We use this to pass a mock worker for testing purposes - resolvedWorkerPath = workerPath; - } else { - // By default, we load ./worker.js but can't rely on the working dir to find it - const dirname = path.dirname(fileURLToPath(import.meta.url)); - resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); - } - const workers = workerpool.pool(resolvedWorkerPath); + // TODO we want to get right down to this + + // Create the internal API + const api = createApi(); + + // Return the external API + return { + execute: api.execute, + listen: api.listen, + }; + + // let resolvedWorkerPath; + // if (workerPath) { + // // If a path to the worker has been passed in, just use it verbatim + // // We use this to pass a mock worker for testing purposes + // resolvedWorkerPath = workerPath; + // } else { + // // By default, we load ./worker.js but can't rely on the working dir to find it + // const dirname = path.dirname(fileURLToPath(import.meta.url)); + // resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + // } + // const workers = workerpool.pool(resolvedWorkerPath); const events = new EventEmitter(); @@ -82,68 +93,68 @@ const createRTM = function (serverId?: string, options: RTMOptions = {}) { } logger.info('repoDir set to ', repoDir); - const onWorkflowStarted = (workflowId: string, threadId: number) => { - logger.info('starting workflow ', workflowId); - const workflow = allWorkflows.get(workflowId)!; - - if (workflow.startTime) { - // TODO this shouldn't throw.. but what do we do? - // We shouldn't run a workflow that's been run - // Every workflow should have a unique id - // maybe the RTM doesn't care about this - throw new Error(`Workflow with id ${workflowId} is already started`); - } - workflow.startTime = new Date().getTime(); - workflow.duration = -1; - workflow.threadId = threadId; - activeWorkflows.push(workflowId); - - // forward the event on to any external listeners - events.emit(e.WORKFLOW_START, { - workflowId, - // Should we publish anything else here? - }); - }; - - const completeWorkflow = (workflowId: string, state: any) => { - logger.success('complete workflow ', workflowId); - logger.info(state); - if (!allWorkflows.has(workflowId)) { - throw new Error(`Workflow with id ${workflowId} is not defined`); - } - const workflow = allWorkflows.get(workflowId)!; - workflow.status = 'done'; - workflow.result = state; - workflow.duration = new Date().getTime() - workflow.startTime!; - const idx = activeWorkflows.findIndex((id) => id === workflowId); - activeWorkflows.splice(idx, 1); - - // forward the event on to any external listeners - events.emit(e.WORKFLOW_COMPLETE, { - id: workflowId, - duration: workflow.duration, - state, - }); - }; - - // Catch a log coming out of a job within a workflow - // Includes runtime logging (is this right?) - const onWorkflowLog = (workflowId: string, message: JSONLog) => { - // Seamlessly proxy the log to the local stdout - // TODO runtime logging probably needs to be at info level? - // Debug information is mostly irrelevant for lightning - const newMessage = { - ...message, - // Prefix the job id in all local jobs - // I'm sure there are nicer, more elegant ways of doing this - message: [`[${workflowId}]`, ...message.message], - }; - logger.proxy(newMessage); - events.emit(e.WORKFLOW_LOG, { - workflowId, - message, - }); - }; + // const onWorkflowStarted = (workflowId: string, threadId: number) => { + // logger.info('starting workflow ', workflowId); + // const workflow = allWorkflows.get(workflowId)!; + + // if (workflow.startTime) { + // // TODO this shouldn't throw.. but what do we do? + // // We shouldn't run a workflow that's been run + // // Every workflow should have a unique id + // // maybe the RTM doesn't care about this + // throw new Error(`Workflow with id ${workflowId} is already started`); + // } + // workflow.startTime = new Date().getTime(); + // workflow.duration = -1; + // workflow.threadId = threadId; + // activeWorkflows.push(workflowId); + + // // forward the event on to any external listeners + // events.emit(e.WORKFLOW_START, { + // workflowId, + // // Should we publish anything else here? + // }); + // }; + + // const completeWorkflow = (workflowId: string, state: any) => { + // logger.success('complete workflow ', workflowId); + // logger.info(state); + // if (!allWorkflows.has(workflowId)) { + // throw new Error(`Workflow with id ${workflowId} is not defined`); + // } + // const workflow = allWorkflows.get(workflowId)!; + // workflow.status = 'done'; + // workflow.result = state; + // workflow.duration = new Date().getTime() - workflow.startTime!; + // const idx = activeWorkflows.findIndex((id) => id === workflowId); + // activeWorkflows.splice(idx, 1); + + // // forward the event on to any external listeners + // events.emit(e.WORKFLOW_COMPLETE, { + // id: workflowId, + // duration: workflow.duration, + // state, + // }); + // }; + + // // Catch a log coming out of a job within a workflow + // // Includes runtime logging (is this right?) + // const onWorkflowLog = (workflowId: string, message: JSONLog) => { + // // Seamlessly proxy the log to the local stdout + // // TODO runtime logging probably needs to be at info level? + // // Debug information is mostly irrelevant for lightning + // const newMessage = { + // ...message, + // // Prefix the job id in all local jobs + // // I'm sure there are nicer, more elegant ways of doing this + // message: [`[${workflowId}]`, ...message.message], + // }; + // logger.proxy(newMessage); + // events.emit(e.WORKFLOW_LOG, { + // workflowId, + // message, + // }); + // }; // How much of this happens inside the worker? // Shoud the main thread handle compilation? Has to if we want to cache diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 0daf791f0..0e21b6a29 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -1,4 +1,9 @@ -import { JSONLog } from '@openfn/logger'; +// TODO remove ths file in favour of types + +// TODO mayberename event constants +// import { JSONLog } from '@openfn/logger'; + +// Top level API events - these are what the engine publishes export const WORKFLOW_START = 'workflow-start'; @@ -8,6 +13,9 @@ export const WORKFLOW_ERROR = 'workflow-error'; export const WORKFLOW_LOG = 'workflow-log'; +// Internal runtime events - these are what the worker thread publishes +// to the engine + type State = any; // TODO export type AcceptWorkflowEvent = { @@ -37,5 +45,5 @@ export type LogWorkflowEvent = { export type WorkflowEvent = | AcceptWorkflowEvent | CompleteWorkflowEvent - | ErrWorkflowEvent - | LogWorkflowEvent; + | ErrWorkflowEvent; +// | LogWorkflowEvent; diff --git a/packages/engine-multi/src/runners/execute.ts b/packages/engine-multi/src/runners/execute.ts deleted file mode 100644 index 783104dc7..000000000 --- a/packages/engine-multi/src/runners/execute.ts +++ /dev/null @@ -1,34 +0,0 @@ -// Execute a compiled workflow -import * as e from '../events'; -import { ModulePaths } from './autoinstall'; - -// A lot of callbacks needed here -// Is it better to just return the handler? -// But then this function really isn't doing so much -// (I guess that's true anyway) -const execute = (context: any, adaptorPaths: ModulePaths, events: any) => { - const { workers, logger, plan } = context; - const { start, log } = events; - return new Promise((resolve) => - workers - .exec('run', [plan, adaptorPaths], { - on: ({ type, ...args }: e.WorkflowEvent) => { - if (type === e.WORKFLOW_START) { - const { workflowId, threadId } = args as e.AcceptWorkflowEvent; - start?.(workflowId, threadId); - } else if (type === e.WORKFLOW_COMPLETE) { - const { state } = args as e.CompleteWorkflowEvent; - resolve(state); - } else if (type === e.WORKFLOW_LOG) { - const { workflowId, message } = args as e.LogWorkflowEvent; - log(workflowId, message); - } - }, - }) - .catch((e: any) => { - logger.error(e); - }) - ); -}; - -export default execute; diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index 7086ec3bf..c37b4706f 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -1,7 +1,10 @@ // ok first of allI want to capture the key interfaces - +import { JSONLog, Logger } from '@openfn/logger'; import { ExecutionPlan } from '@openfn/runtime'; +import type { EventEmitter } from 'node:events'; +import workerpool from 'workerpool'; +// These are the external events published but he api and listen type WorkflowStartEvent = 'workflow-start'; type WorkflowStartPayload = { workflowId: string; @@ -63,6 +66,25 @@ type EventPayloadLookup = { [LogEvent]: LogPayload; }; +// These are events from the internal worker (& runtime) + +export type WORKER_START = 'worker-start'; +export type WorkerStartPayload = { + threadId: string; + workflowId: string; +}; + +export type WORKER_COMPLETE = 'worker-complete'; +export type WorkerCompletePayload = { + workflowId: string; + state: any; +}; + +export type WORKER_LOG = 'worker-log'; +export type WorkerLogPayload = JSONLog & { + workflowId: string; +}; + type EventHandler = ( event: EventPayloadLookup[T] ) => void; @@ -79,6 +101,30 @@ type ExecuteOptions = { noCompile: any; // skip compilation (useful in test) }; +export type WorkflowState = { + id: string; + name?: string; // TODO what is name? this is irrelevant? + status: 'pending' | 'running' | 'done' | 'err'; + threadId?: string; + startTime?: number; + duration?: number; + error?: string; + result?: any; // State + plan: ExecutionPlan; // this doesn't include options + options: any; // TODO this is general engine options and workflow options +}; + +// this is the internal engine API +export interface EngineAPI extends EventEmitter { + logger: Logger; + + callWorker: ( + task: string, + args: any[] = [], + events: any = {} + ) => workerpool.Promise; +} + interface RuntimeEngine extends EventEmitter { //id: string // human readable instance id // actually I think the id is on the worker, not the engine @@ -97,5 +143,5 @@ interface RuntimeEngine extends EventEmitter { options: ExecuteOptions = {} ); - // TODO my want some maintenance APIs, like getSatus. idk + // TODO my want some maintenance APIs, like getStatus. idk } diff --git a/packages/engine-multi/src/worker-helper.ts b/packages/engine-multi/src/worker-helper.ts index d5f032639..88c52982a 100644 --- a/packages/engine-multi/src/worker-helper.ts +++ b/packages/engine-multi/src/worker-helper.ts @@ -12,11 +12,11 @@ function publish(event: e.WorkflowEvent) { } export const createLoggers = (workflowId: string) => { - const log = (jsonLog: string) => { + const log = (message: JSONLog) => { publish({ workflowId, type: e.WORKFLOW_LOG, - message: JSON.parse(jsonLog) as JSONLog, + message, }); }; diff --git a/packages/engine-multi/src/worker.ts b/packages/engine-multi/src/worker.ts index b4f1944b4..5bc2fa342 100644 --- a/packages/engine-multi/src/worker.ts +++ b/packages/engine-multi/src/worker.ts @@ -1,4 +1,5 @@ // Runs inside the worker +// TODO maybe this is // Dedicated worker for running jobs // Security thoughts: the process inherits the node command arguments diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts new file mode 100644 index 000000000..6e6cd0070 --- /dev/null +++ b/packages/engine-multi/test/api.test.ts @@ -0,0 +1,20 @@ +import test from 'ava'; + +import createAPI from '../src/api'; + +// thes are tests on the api functions generally + +// no need to test the event stuff - startworkflow etc +// maybe we can check the keys exist, although we'll quickly know if we dont + +test.todo('execute'); +test.todo('execute should return an event emitter'); +test.todo('execute should proxy events'); +test.todo('listen'); +test.todo('log'); + +test('callWorker', (t) => { + const api = createAPI(); + + t.pass(); +}); diff --git a/packages/engine-multi/test/runners/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts similarity index 54% rename from packages/engine-multi/test/runners/autoinstall.test.ts rename to packages/engine-multi/test/api/autoinstall.test.ts index 314d41750..d9685cea7 100644 --- a/packages/engine-multi/test/runners/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -1,7 +1,15 @@ import test from 'ava'; -import autoinstall, { identifyAdaptors } from '../../src/runners/autoinstall'; +import { createMockLogger } from '@openfn/logger'; -const mockIsInstalled = (pkg) => async (specifier: string) => { +import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; +import { EngineAPI, WorkflowState } from '../../src/types'; + +type PackageJson = { + name: string; + [x: string]: any; +}; + +const mockIsInstalled = (pkg: PackageJson) => async (specifier: string) => { const alias = specifier.split('@').join('_'); return pkg.dependencies.hasOwnProperty(alias); }; @@ -11,6 +19,12 @@ const mockIsInstalled = (pkg) => async (specifier: string) => { const mockHandleInstall = async (specifier: string): Promise => new Promise((r) => r()).then(); +const mockLogger = createMockLogger(); + +const api = { + logger: mockLogger, +} as unknown as EngineAPI; + test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ name: 'repo', @@ -61,7 +75,7 @@ test('identifyAdaptors: pick out adaptors and remove duplicates', (t) => { }); // This doesn't do anything except check that the mocks are installed -test('autoinstall: should call both mock functions', async (t) => { +test.serial('autoinstall: should call both mock functions', async (t) => { let didCallIsInstalled = false; let didCallInstall = true; @@ -74,77 +88,77 @@ test('autoinstall: should call both mock functions', async (t) => { return; }; - await autoinstall({ - options: { - handleInstall: mockInstall, - handleIsInstalled: mockIsInstalled, - }, + const state = { plan: { jobs: [{ adaptor: 'x@1.0.0' }], }, - }); + } as WorkflowState; + + const options = { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }; + + await autoinstall(api, state, options); t.true(didCallIsInstalled); t.true(didCallInstall); }); -// TODO a problem with this test is that pending state is shared across tests -test('autoinstall: only call install once if there are two concurrent install requests', async (t) => { - let callCount = 0; - - const mockInstall = (specififer: string) => - new Promise((resolve) => { - callCount++; - setTimeout(() => resolve(), 20); - }); - - await Promise.all([ - autoinstall({ - options: { - handleInstall: mockInstall, - handleIsInstalled: async () => false, - }, - plan: { - jobs: [{ adaptor: 'z@1.0.0' }], - }, - }), +test.serial( + 'autoinstall: only call install once if there are two concurrent install requests', + async (t) => { + let callCount = 0; - autoinstall({ - options: { - handleInstall: mockInstall, - handleIsInstalled: async () => false, - }, + const mockInstall = (specififer: string) => + new Promise((resolve) => { + callCount++; + setTimeout(() => resolve(), 20); + }); + + const options = { + handleInstall: mockInstall, + handleIsInstalled: async () => false, + }; + + const state = { plan: { jobs: [{ adaptor: 'z@1.0.0' }], }, - }), - ]); + } as WorkflowState; - t.is(callCount, 1); -}); + await Promise.all([ + autoinstall(api, state, options), + autoinstall(api, state, options), + ]); -test('autoinstall: return a map to modules', async (t) => { - const plan = { - // Note that we have difficulty now if a workflow imports two versions of the same adaptor - jobs: [ - { - adaptor: 'common@1.0.0', - }, - { - adaptor: 'http@1.0.0', - }, - ], - }; + t.is(callCount, 1); + } +); - const result = await autoinstall({ - plan, - options: { - repoDir: 'a/b/c', - skipRepoValidation: true, - handleInstall: async () => true, - handleIsInstalled: async () => false, +test.serial('autoinstall: return a map to modules', async (t) => { + const state = { + plan: { + // Note that we have difficulty now if a workflow imports two versions of the same adaptor + jobs: [ + { + adaptor: 'common@1.0.0', + }, + { + adaptor: 'http@1.0.0', + }, + ], }, - }); + } as WorkflowState; + + const options = { + repoDir: 'a/b/c', + skipRepoValidation: true, + handleInstall: async () => true, + handleIsInstalled: async () => false, + }; + + const result = await autoinstall(api, state, options); t.deepEqual(result, { common: { path: 'a/b/c/node_modules/common_1.0.0' }, diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts new file mode 100644 index 000000000..a4c143d66 --- /dev/null +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -0,0 +1,64 @@ +import test from 'ava'; +import path from 'node:path'; + +import initWorkers, { createWorkers } from '../../src/api/call-worker'; +import { EngineAPI } from '../../src/types'; + +let api = {} as EngineAPI; + +test.before(() => { + const workerPath = path.resolve('test/worker-functions.js'); + initWorkers(api, workerPath); +}); + +test('initWorkers should add a callWorker function', (t) => { + t.assert(typeof api.callWorker === 'function'); +}); + +test('callWorker should return the default result', async (t) => { + const result = await api.callWorker('test'); + t.is(result, 42); +}); + +test('callWorker should return a custom result', async (t) => { + const result = await api.callWorker('test', [84]); + t.is(result, 84); +}); + +test('callWorker should trigger an event callback', async (t) => { + return new Promise((done) => { + const onCallback = ({ result }) => { + t.is(result, 11); + done(); + }; + + api.callWorker('test', [11], { message: onCallback }); + }); +}); + +// Dang, this doesn't work, the worker threads run in the same process +test.skip('callWorker should execute with a different process id', async (t) => { + return new Promise((done) => { + const onCallback = ({ pid }) => { + t.not(process.pid, pid); + done(); + }; + + api.callWorker('test', [], { message: onCallback }); + }); +}); + +test('callWorker should execute in a different process', async (t) => { + return new Promise((done) => { + // @ts-ignore + process.scribble = 'xyz'; + + const onCallback = ({ scribble }) => { + // @ts-ignore + t.not(process.scribble, scribble); + done(); + }; + + api.callWorker('test', [], { message: onCallback }); + }); +}); diff --git a/packages/engine-multi/test/runners/compile.test.ts b/packages/engine-multi/test/api/compile.test.ts similarity index 100% rename from packages/engine-multi/test/runners/compile.test.ts rename to packages/engine-multi/test/api/compile.test.ts diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts new file mode 100644 index 000000000..145790f14 --- /dev/null +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -0,0 +1,124 @@ +import test from 'ava'; +import { EventEmitter } from 'node:events'; + +import * as e from '../../src/events'; +import { createMockLogger } from '@openfn/logger'; +import { log, workflowComplete, workflowStart } from '../../src/api/lifecycle'; +import { EngineAPI, WorkflowState } from '../../src/types'; + +// TODO this probably wants unit testing +// is it even worth mocking it? +const createMockAPI = (): EngineAPI => { + const api = new EventEmitter(); + + Object.assign(api, { + logger: createMockLogger(), + getWorkflowState: () => {}, + setWorkflowState: () => {}, + }); + + return api as EngineAPI; +}; + +test(`workflowStart: emits ${e.WORKFLOW_START}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + + const api = createMockAPI(); + const state = {} as WorkflowState; + const event = { workflowId, threadId: '123' }; + + api.on(e.WORKFLOW_START, (evt) => { + t.deepEqual(evt, { workflowId }); + done(); + }); + + workflowStart(api, state, event); + }); +}); + +test('onWorkflowStart: updates state', (t) => { + const workflowId = 'a'; + + const api = createMockAPI(); + const state = {} as WorkflowState; + const event = { workflowId, threadId: '123' }; + + workflowStart(api, state, event); + + t.is(state.status, 'running'); + t.is(state.duration, -1); + t.is(state.threadId, '123'); + t.truthy(state.startTime); +}); + +test.todo('onWorkflowStart: logs'); +test.todo('onWorkflowStart: throws if the workflow is already started'); + +test(`workflowComplete: emits ${e.WORKFLOW_COMPLETE}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + const result = { a: 777 }; + + const api = createMockAPI(); + const state = { + startTime: Date.now() - 1000, + } as WorkflowState; + + const event = { workflowId, state: result }; + + api.on(e.WORKFLOW_COMPLETE, (evt) => { + t.is(evt.id, workflowId); + t.deepEqual(evt.state, result); + t.assert(evt.duration > 0); + done(); + }); + + workflowComplete(api, state, event); + }); +}); + +test('workflowComplete: updates state', (t) => { + const workflowId = 'a'; + const result = { a: 777 }; + + const api = createMockAPI(); + const state = { + startTime: Date.now() - 1000, + } as WorkflowState; + const event = { workflowId, state: result }; + + workflowComplete(api, state, event); + + t.is(state.status, 'done'); + t.assert(state.duration! > 0); + t.deepEqual(state.result, result); +}); + +test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + + const api = createMockAPI(); + const state = { + id: workflowId, + } as WorkflowState; + + const event = { + level: 'info', + name: 'job', + message: ['oh hai'], + time: Date.now() - 100, + }; + + api.on(e.WORKFLOW_LOG, (evt) => { + t.deepEqual(evt, { + workflowId: state.id, + ...event, + }); + done(); + }); + + log(api, state, event); + }); +}); diff --git a/packages/engine-multi/test/worker-functions.js b/packages/engine-multi/test/worker-functions.js new file mode 100644 index 000000000..497d8938c --- /dev/null +++ b/packages/engine-multi/test/worker-functions.js @@ -0,0 +1,16 @@ +import workerpool from 'workerpool'; + +workerpool.worker({ + test: (result = 42) => { + const { pid, scribble } = process; + + workerpool.workerEmit({ + type: 'message', + result, + pid, + scribble, + }); + + return result; + }, +}); From 30b70a0bed26c02e332d146b70c1c241e0ac8aa1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 10 Oct 2023 18:25:15 +0100 Subject: [PATCH 132/232] Implement and test execute --- packages/engine-multi/package.json | 3 +- packages/engine-multi/src/api/autoinstall.ts | 4 +- packages/engine-multi/src/api/compile.ts | 24 ++-- packages/engine-multi/src/api/execute.ts | 18 ++- packages/engine-multi/src/api/lifecycle.ts | 6 +- packages/engine-multi/src/engine.ts | 8 +- packages/engine-multi/src/events.ts | 2 +- packages/engine-multi/src/types.d.ts | 4 +- packages/engine-multi/src/worker-helper.ts | 6 +- .../engine-multi/test/api/execute.test.ts | 126 ++++++++++++++++++ .../engine-multi/test/api/lifecycle.test.ts | 15 ++- .../engine-multi/test/worker-functions.js | 32 +++++ 12 files changed, 213 insertions(+), 35 deletions(-) create mode 100644 packages/engine-multi/test/api/execute.test.ts diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 267359bd1..7f0f30877 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -8,7 +8,8 @@ "scripts": { "test": "pnpm ava", "_test:types": "pnpm tsc --noEmit --project tsconfig.json", - "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", + "_build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", + "build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch" }, "author": "Open Function Group ", diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 5f52db05c..ac8a6e6e0 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -12,7 +12,7 @@ import type { Logger } from '@openfn/logger'; import { EngineAPI, WorkflowState } from '../types'; // none of these options should be on the plan actually -type Options = { +export type AutoinstallOptions = { repoDir?: string; skipRepoValidation?: boolean; handleInstall?(fn: string, repoDir: string, logger: Logger): Promise; @@ -28,7 +28,7 @@ const pending: Record> = {}; const autoinstall = async ( api: EngineAPI, state: WorkflowState, - options: Options + options: AutoinstallOptions ): Promise => { const { logger } = api; const { plan } = state; diff --git a/packages/engine-multi/src/api/compile.ts b/packages/engine-multi/src/api/compile.ts index ba58c93eb..1303f3f39 100644 --- a/packages/engine-multi/src/api/compile.ts +++ b/packages/engine-multi/src/api/compile.ts @@ -6,6 +6,7 @@ import type { Logger } from '@openfn/logger'; import compile, { preloadAdaptorExports } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; import { EngineAPI, WorkflowState } from '../types'; +import { RTEOptions } from '../engine'; // TODO this compiler is going to change anyway to run just in time // the runtime will have an onCompile hook @@ -13,22 +14,25 @@ import { EngineAPI, WorkflowState } from '../types'; export default async ( api: EngineAPI, state: WorkflowState, - options: { repoDir: string } + options: Pick = {} ) => { const { logger } = api; const { plan } = state; - const { repoDir } = options; + const { repoDir, noCompile } = options; - for (const job of plan.jobs) { - if (job.expression) { - job.expression = await compileJob( - job.expression as string, - job.adaptor, // TODO need to expand this. Or do I? - repoDir, - logger - ); + if (!noCompile) { + for (const job of plan.jobs) { + if (job.expression) { + job.expression = await compileJob( + job.expression as string, + job.adaptor, // TODO need to expand this. Or do I? + repoDir, + logger + ); + } } } + return plan; }; diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index ea89feb1f..f39f77d14 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -12,8 +12,12 @@ import { workflowStart, workflowComplete, log } from './lifecycle'; // Is it better to just return the handler? // But then this function really isn't doing so much // (I guess that's true anyway) -const execute = async (api: EngineAPI, state: WorkflowState, options) => { - const adaptorPaths = await autoinstall(api, state, options); +const execute = async ( + api: EngineAPI, + state: WorkflowState, + options: RTEOptions +) => { + const adaptorPaths = await autoinstall(api, state, options.autoinstall); await compile(api, state, options); const events = { @@ -28,10 +32,12 @@ const execute = async (api: EngineAPI, state: WorkflowState, options) => { }, }; - return api.callWorker('run', [plan, adaptorPaths], events).catch((e) => { - // TODO what about errors then? - api.logger.error(e); - }); + return api + .callWorker('run', [state.plan, adaptorPaths], events) + .catch((e) => { + // TODO what about errors then? + api.logger.error(e); + }); }; export default execute; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index d84bdffbb..54c2f3caa 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -67,7 +67,7 @@ export const workflowComplete = ( // forward the event on to any external listeners api.emit(WORKFLOW_COMPLETE, { - id: workflowId, + workflowId: workflowId, duration: state.duration, state: result, }); @@ -86,11 +86,11 @@ export const log = ( // // I'm sure there are nicer, more elegant ways of doing this // message: [`[${workflowId}]`, ...message.message], // }; - api.logger.proxy(event); + api.logger.proxy(event.message); api.emit(WORKFLOW_LOG, { workflowId: id, - ...event, + ...event.message, }); }; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index f3392ecef..6f5fe0e0c 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -9,7 +9,7 @@ import createLogger, { JSONLog, Logger } from '@openfn/logger'; import * as e from './events'; import compile from './api/compile'; import execute from './runners/execute'; -import autoinstall from './api/autoinstall'; +import autoinstall, { AutoinstallOptions } from './api/autoinstall'; export type State = any; // TODO I want a nice state def with generics @@ -37,15 +37,17 @@ export type LazyResolvers = { expressions?: Resolver; }; -type RTMOptions = { +export type RTEOptions = { resolvers?: LazyResolvers; logger?: Logger; workerPath?: string; repoDir?: string; noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? + + autoinstall: AutoinstallOptions; }; -const createRTM = function (serverId?: string, options: RTMOptions = {}) { +const createRTM = function (serverId?: string, options: RTEOptions = {}) { const { noCompile } = options; let { repoDir, workerPath } = options; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 0e21b6a29..1d04bf2b6 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -1,7 +1,7 @@ // TODO remove ths file in favour of types // TODO mayberename event constants -// import { JSONLog } from '@openfn/logger'; +import { JSONLog } from '@openfn/logger'; // Top level API events - these are what the engine publishes diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index c37b4706f..5e599a56e 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -80,9 +80,11 @@ export type WorkerCompletePayload = { state: any; }; +// TODO confusion over this and events.ts export type WORKER_LOG = 'worker-log'; -export type WorkerLogPayload = JSONLog & { +export type WorkerLogPayload = { workflowId: string; + message: JSONLog; }; type EventHandler = ( diff --git a/packages/engine-multi/src/worker-helper.ts b/packages/engine-multi/src/worker-helper.ts index 88c52982a..0ad9cbc5a 100644 --- a/packages/engine-multi/src/worker-helper.ts +++ b/packages/engine-multi/src/worker-helper.ts @@ -12,11 +12,13 @@ function publish(event: e.WorkflowEvent) { } export const createLoggers = (workflowId: string) => { - const log = (message: JSONLog) => { + const log = (message: string) => { + // hmm, the json log stringifies the message + // i don't really want it to do that publish({ workflowId, type: e.WORKFLOW_LOG, - message, + message: JSON.parse(message), }); }; diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts new file mode 100644 index 000000000..f10ffda79 --- /dev/null +++ b/packages/engine-multi/test/api/execute.test.ts @@ -0,0 +1,126 @@ +import path from 'node:path'; +import test from 'ava'; +import { EventEmitter } from 'node:events'; +import { EngineAPI } from '../../src/types'; +import initWorkers from '../../src/api/call-worker'; +import execute from '../../src/api/execute'; +import { createMockLogger } from '@openfn/logger'; +import { + WORKFLOW_COMPLETE, + WORKFLOW_LOG, + WORKFLOW_START, +} from '../../src/events'; + +const workerPath = path.resolve('dist/mock-worker.js'); + +// Mock API object +let api; + +test.before(() => { + api = new EventEmitter() as EngineAPI; + api.logger = createMockLogger(); + + initWorkers(api, workerPath); +}); + +const plan = { + id: 'x', + jobs: [ + { + // this will basically be evalled + expression: '() => 22', + }, + ], +}; + +const options = { + noCompile: true, + autoinstall: { + handleInstall: async () => false, + handleIsInstalled: async () => false, + }, +}; + +test.serial('execute should run a job and return the result', async (t) => { + const state = { + plan, + }; + + const options = { + noCompile: true, + autoinstall: { + handleInstall: async () => false, + handleIsInstalled: async () => false, + }, + }; + + const result = await execute(api, state, options); + t.is(result, 22); +}); + +// we can check the state object after each of these is returned +test.serial('should emit a workflow-start event', async (t) => { + let workflowStart; + + api.once(WORKFLOW_START, (evt) => (workflowStart = evt)); + + const state = { + plan, + }; + + await execute(api, state, options); + + // No need to do a deep test of the event payload here + t.is(workflowStart.workflowId, 'x'); +}); + +test.serial('should emit a workflow-complete event', async (t) => { + let workflowComplete; + + api.once(WORKFLOW_COMPLETE, (evt) => (workflowComplete = evt)); + + const state = { + plan, + }; + + await execute(api, state, options); + + t.is(workflowComplete.workflowId, 'x'); + t.is(workflowComplete.state, 22); +}); + +test.serial('should emit a log event', async (t) => { + let workflowLog; + + api.once(WORKFLOW_LOG, (evt) => (workflowLog = evt)); + + const plan = { + id: 'y', + jobs: [ + { + expression: '() => { console.log("hi"); return 33 }', + }, + ], + }; + const state = { + id: 'y', + plan, + }; + + await execute(api, state, options); + + t.is(workflowLog.workflowId, 'y'); + t.is(workflowLog.message[0], 'hi'); + t.is(workflowLog.level, 'info'); +}); + +// how will we test compilation? +// compile will call the actual runtime +// maybe that's fine? +test.todo('should compile'); + +// what are we actually testing here? +// ideally we wouldc ensure that autoinstall is called with the corret aguments +// we can pass in mock autoinstall handlers and ensure they're invoked +// maybe we can also test that the correct adaptorpaths are passed in to execute +test.todo('should autoinstall'); diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 145790f14..55f4043e3 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -68,7 +68,7 @@ test(`workflowComplete: emits ${e.WORKFLOW_COMPLETE}`, (t) => { const event = { workflowId, state: result }; api.on(e.WORKFLOW_COMPLETE, (evt) => { - t.is(evt.id, workflowId); + t.is(evt.workflowId, workflowId); t.deepEqual(evt.state, result); t.assert(evt.duration > 0); done(); @@ -105,16 +105,19 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { } as WorkflowState; const event = { - level: 'info', - name: 'job', - message: ['oh hai'], - time: Date.now() - 100, + workflowId, + message: { + level: 'info', + name: 'job', + message: ['oh hai'], + time: Date.now() - 100, + }, }; api.on(e.WORKFLOW_LOG, (evt) => { t.deepEqual(evt, { workflowId: state.id, - ...event, + ...event.message, }); done(); }); diff --git a/packages/engine-multi/test/worker-functions.js b/packages/engine-multi/test/worker-functions.js index 497d8938c..028690413 100644 --- a/packages/engine-multi/test/worker-functions.js +++ b/packages/engine-multi/test/worker-functions.js @@ -1,4 +1,5 @@ import workerpool from 'workerpool'; +import { threadId } from 'node:worker_threads'; workerpool.worker({ test: (result = 42) => { @@ -13,4 +14,35 @@ workerpool.worker({ return result; }, + // // very basic simulation of a run + // hmm, don't use this, use mock worker instead + // run: (plan, adaptorPaths) => { + // workerpool.workerEmit({ type: e.WORKFLOW_START, workflowId, threadId }); + // try { + // // TODO + // // workerpool.workerEmit({ + // // type: e.WORKFLOW_LOG, + // // }); + + // // or something + // const result = eval(plan); + + // workerpool.workerEmit({ + // type: e.WORKFLOW_COMPLETE, + // workflowId, + // state: result, + // }); + + // // For tests + // return result; + // } catch (err) { + // console.error(err); + // // @ts-ignore TODO sort out error typing + // workerpool.workerEmit({ + // type: e.WORKFLOW_ERROR, + // workflowId, + // message: err.message, + // }); + // } + // }, }); From 95a851147ff956a8a9475dc473501669dfa11561 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 11 Oct 2023 12:55:34 +0100 Subject: [PATCH 133/232] engine: sort out api and engine, add unit tests on internal engine --- packages/engine-multi/src/__api.ts | 123 +++++++ packages/engine-multi/src/__engine.ts | 207 +++++++++++ packages/engine-multi/src/api.ts | 169 ++++----- packages/engine-multi/src/api/call-worker.ts | 3 +- packages/engine-multi/src/api/compile.ts | 2 +- packages/engine-multi/src/api/lifecycle.ts | 3 +- packages/engine-multi/src/engine.ts | 346 ++++++++---------- packages/engine-multi/src/index.ts | 4 +- packages/engine-multi/test/api.test.ts | 5 +- packages/engine-multi/test/engine.test.ts | 185 ++++++---- .../engine-multi/test/worker-functions.js | 59 ++- 11 files changed, 705 insertions(+), 401 deletions(-) create mode 100644 packages/engine-multi/src/__api.ts create mode 100644 packages/engine-multi/src/__engine.ts diff --git a/packages/engine-multi/src/__api.ts b/packages/engine-multi/src/__api.ts new file mode 100644 index 000000000..9b5f58112 --- /dev/null +++ b/packages/engine-multi/src/__api.ts @@ -0,0 +1,123 @@ +import createLogger, { JSONLog, Logger } from '@openfn/logger'; +import { EngineAPI, EngineEvents, EventHandler } from './types'; +import { EventEmitter } from 'node:events'; +import { WORKFLOW_COMPLETE, WORKFLOW_START } from './events'; + +import initWorkers from './api/call-worker'; +import { RTEOptions } from './__engine'; + +// For each workflow, create an API object with its own event emitter +// this is a bt wierd - what if the emitter went on state instead? +const createWorkflowEvents = (api: EngineAPI) => { + //create a bespoke event emitter + const events = new EventEmitter(); + + // TODO need this in closure really + // listeners[workflowId] = events; + + // proxy all events to the main emitter + // uh actually there may be no point in this + function proxy(event: string) { + events.on(event, (evt) => { + // ensure the attempt id is on the event + evt.workflowId = workflowId; + const newEvt = { + ...evt, + workflowId: workflowId, + }; + + api.emit(event, newEvt); + }); + } + proxy(WORKFLOW_START); + proxy(WORKFLOW_COMPLETE); + proxy(JOB_START); + proxy(JOB_COMPLETE); + proxy(LOG); + + return events; +}; + +// This creates the internal API +// tbh this is actually the engine, right, this is where stuff happens +// the api file is more about the public api I think +const createAPI = (options: RTEOptions) => { + const listeners = {}; + + // but this is more like an internal api, right? + // maybe this is like the workflow context + const state = { + workflows: {}, + }; // TODO maybe this sits in another file + // the state has apis for getting/setting workflow state + // maybe actually each execute setsup its own state object + + // TODO I think there's an internal and external API + // api is the external thing that other people call + // engine is the internal thing + const api = new EventEmitter() as EngineAPI; + + api.logger = options.logger || createLogger('RTE', { level: 'debug' }); + + api.registerWorkflow = (state) => { + state.workflows[plan.id] = state; + }; + + // what if this returns a bespoke event listener? + // i don't need to to execute(); listen, i can just excute + // it's kinda slick but you still need two lines of code and it doesn't buy anyone anything + // also this is nevver gonna get used externally so it doesn't need to be slick + api.execute = (executionPlan) => { + const workflowId = plan.id; + + // Pull options out of the plan so that all options are in one place + const { options, ...plan } = executionPlan; + + // initial state for this workflow run + // TODO let's create a util function for this (nice for testing) + const state = createState(plan); + // the engine does need to be able to report on the state of each workflow + api.registerWorkflow(state); + + const events = createWorkflowEvents(api); + + listeners[workflowId] = events; + + // this context API thing is the internal api / engine + // each has a bespoke event emitter but otherwise a common interface + const contextAPI: EngineAPI = { + ...api, + ...events, + }; + + execute(contextAPI, state, options); + + // return the event emitter (not the full engine API though) + return events; + }; + + // // how will this actually work? + // api.listen = ( + // attemptId: string, + // listeners: Record + // ) => { + // // const handler = (eventName) => { + // // if () + // // } + // const events = listeners[workflowId]; + // for (const evt of listeners) { + // events.on(evt, listeners[evt]); + // } + + // // TODO return unsubscribe handle + // }; + + // we can have global reporting like this + api.getStatus = (workflowId) => state.workflows[workflowId].status; + + initWorkers(api); + + return api; +}; + +export default createAPI; diff --git a/packages/engine-multi/src/__engine.ts b/packages/engine-multi/src/__engine.ts new file mode 100644 index 000000000..6d3fad724 --- /dev/null +++ b/packages/engine-multi/src/__engine.ts @@ -0,0 +1,207 @@ +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'url'; +import { EventEmitter } from 'node:events'; +import workerpool from 'workerpool'; +import { ExecutionPlan } from '@openfn/runtime'; +import createLogger, { JSONLog, Logger } from '@openfn/logger'; + +import * as e from './events'; +import compile from './api/compile'; +import execute from './runners/execute'; +import autoinstall, { AutoinstallOptions } from './api/autoinstall'; + +export type State = any; // TODO I want a nice state def with generics + +// Archive of every workflow we've run +// Fine to just keep in memory for now +type WorkflowStats = { + id: string; + name?: string; // TODO what is name? this is irrelevant? + status: 'pending' | 'done' | 'err'; + startTime?: number; + threadId?: number; + duration?: number; + error?: string; + result?: any; // State + plan: ExecutionPlan; +}; + +type Resolver = (id: string) => Promise; + +// A list of helper functions which basically resolve ids into JSON +// to lazy load assets +export type LazyResolvers = { + credentials?: Resolver; + state?: Resolver; + expressions?: Resolver; +}; + +export type RTEOptions = { + resolvers?: LazyResolvers; + logger?: Logger; + workerPath?: string; + repoDir?: string; + noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? + + autoinstall: AutoinstallOptions; +}; + +const createRTM = function (serverId?: string, options: RTEOptions = {}) { + const { noCompile } = options; + let { repoDir, workerPath } = options; + + const id = serverId || crypto.randomUUID(); + const logger = options.logger || createLogger('RTM', { level: 'debug' }); + + const allWorkflows: Map = new Map(); + const activeWorkflows: string[] = []; + + // TODO we want to get right down to this + + // Create the internal API + const api = createApi(options); + + // Return the external API + return { + execute: api.execute, + listen: api.listen, + }; + + // let resolvedWorkerPath; + // if (workerPath) { + // // If a path to the worker has been passed in, just use it verbatim + // // We use this to pass a mock worker for testing purposes + // resolvedWorkerPath = workerPath; + // } else { + // // By default, we load ./worker.js but can't rely on the working dir to find it + // const dirname = path.dirname(fileURLToPath(import.meta.url)); + // resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + // } + // const workers = workerpool.pool(resolvedWorkerPath); + + const events = new EventEmitter(); + + if (!repoDir) { + if (process.env.OPENFN_RTE_REPO_DIR) { + repoDir = process.env.OPENFN_RTE_REPO_DIR; + } else { + repoDir = '/tmp/openfn/repo'; + logger.warn('Using default repodir'); + logger.warn( + 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' + ); + } + } + logger.info('repoDir set to ', repoDir); + + // const onWorkflowStarted = (workflowId: string, threadId: number) => { + // logger.info('starting workflow ', workflowId); + // const workflow = allWorkflows.get(workflowId)!; + + // if (workflow.startTime) { + // // TODO this shouldn't throw.. but what do we do? + // // We shouldn't run a workflow that's been run + // // Every workflow should have a unique id + // // maybe the RTM doesn't care about this + // throw new Error(`Workflow with id ${workflowId} is already started`); + // } + // workflow.startTime = new Date().getTime(); + // workflow.duration = -1; + // workflow.threadId = threadId; + // activeWorkflows.push(workflowId); + + // // forward the event on to any external listeners + // events.emit(e.WORKFLOW_START, { + // workflowId, + // // Should we publish anything else here? + // }); + // }; + + // const completeWorkflow = (workflowId: string, state: any) => { + // logger.success('complete workflow ', workflowId); + // logger.info(state); + // if (!allWorkflows.has(workflowId)) { + // throw new Error(`Workflow with id ${workflowId} is not defined`); + // } + // const workflow = allWorkflows.get(workflowId)!; + // workflow.status = 'done'; + // workflow.result = state; + // workflow.duration = new Date().getTime() - workflow.startTime!; + // const idx = activeWorkflows.findIndex((id) => id === workflowId); + // activeWorkflows.splice(idx, 1); + + // // forward the event on to any external listeners + // events.emit(e.WORKFLOW_COMPLETE, { + // id: workflowId, + // duration: workflow.duration, + // state, + // }); + // }; + + // // Catch a log coming out of a job within a workflow + // // Includes runtime logging (is this right?) + // const onWorkflowLog = (workflowId: string, message: JSONLog) => { + // // Seamlessly proxy the log to the local stdout + // // TODO runtime logging probably needs to be at info level? + // // Debug information is mostly irrelevant for lightning + // const newMessage = { + // ...message, + // // Prefix the job id in all local jobs + // // I'm sure there are nicer, more elegant ways of doing this + // message: [`[${workflowId}]`, ...message.message], + // }; + // logger.proxy(newMessage); + // events.emit(e.WORKFLOW_LOG, { + // workflowId, + // message, + // }); + // }; + + // How much of this happens inside the worker? + // Shoud the main thread handle compilation? Has to if we want to cache + // Unless we create a dedicated compiler worker + // TODO error handling, timeout + const handleExecute = async (plan: ExecutionPlan) => { + const options = { + repoDir, + }; + const context = { plan, logger, workers, options /* api */ }; + + logger.debug('Executing workflow ', plan.id); + + allWorkflows.set(plan.id!, { + id: plan.id!, + status: 'pending', + plan, + }); + + const adaptorPaths = await autoinstall(context); + + if (!noCompile) { + context.plan = await compile(context); + } + + logger.debug('workflow compiled ', plan.id); + const result = await execute(context, adaptorPaths, { + start: onWorkflowStarted, + log: onWorkflowLog, + }); + completeWorkflow(plan.id!, result); + + logger.debug('finished executing workflow ', plan.id); + // Return the result + // Note that the mock doesn't behave like ths + // And tbf I don't think we should keep the promise open - there's no point? + return result; + }; + + return { + id, + on: (type: string, fn: (...args: any[]) => void) => events.on(type, fn), + once: (type: string, fn: (...args: any[]) => void) => events.once(type, fn), + execute: handleExecute, + }; +}; + +export default createRTM; diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 243348f7f..78f3f5130 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -1,119 +1,84 @@ +// Creates the public/external API to the runtime + +import path from 'node:path'; +import crypto from 'node:crypto'; +import { ExecutionPlan } from '@openfn/runtime'; import createLogger, { JSONLog, Logger } from '@openfn/logger'; -import { EngineAPI, EngineEvents, EventHandler } from './types'; -import { EventEmitter } from 'node:events'; -import { WORKFLOW_COMPLETE, WORKFLOW_START } from './events'; - -import initWorkers from './api/call-worker'; - -// For each workflow, create an API object with its own event emitter -// this is a bt wierd - what if the emitter went on state instead? -const createWorkflowEvents = (api: EngineAPI) => { - //create a bespoke event emitter - const events = new EventEmitter(); - - // TODO need this in closure really - // listeners[workflowId] = events; - - // proxy all events to the main emitter - // uh actually there may be no point in this - function proxy(event: string) { - events.on(event, (evt) => { - // ensure the attempt id is on the event - evt.workflowId = workflowId; - const newEvt = { - ...evt, - workflowId: workflowId, - }; - - api.emit(event, newEvt); - }); - } - proxy(WORKFLOW_START); - proxy(WORKFLOW_COMPLETE); - proxy(JOB_START); - proxy(JOB_COMPLETE); - proxy(LOG); - return events; +import type { AutoinstallOptions } from './api/autoinstall'; + +export type State = any; // TODO I want a nice state def with generics + +// Archive of every workflow we've run +// Fine to just keep in memory for now +type WorkflowStats = { + id: string; + name?: string; // TODO what is name? this is irrelevant? + status: 'pending' | 'done' | 'err'; + startTime?: number; + threadId?: number; + duration?: number; + error?: string; + result?: any; // State + plan: ExecutionPlan; }; -const createAPI = (repoDir: string, options) => { - const listeners = {}; - - // but this is more like an internal api, right? - // maybe this is like the workflow context - const state = { - workflows: {}, - }; // TODO maybe this sits in another file - // the state has apis for getting/setting workflow state - // maybe actually each execute setsup its own state object - - // TODO I think there's an internal and external API - // api is the external thing that other people call - // engine is the internal thing - const api = new EventEmitter() as EngineAPI; +type Resolver = (id: string) => Promise; - api.logger = options.logger || createLogger('RTE', { level: 'debug' }); - - api.registerWorkflow = (state) => { - state.workflows[plan.id] = state; - }; +// A list of helper functions which basically resolve ids into JSON +// to lazy load assets +export type LazyResolvers = { + credentials?: Resolver; + state?: Resolver; + expressions?: Resolver; +}; - // what if this returns a bespoke event listener? - // i don't need to to execute(); listen, i can just excute - // it's kinda slick but you still need two lines of code and it doesn't buy anyone anything - // also this is nevver gonna get used externally so it doesn't need to be slick - api.execute = (executionPlan) => { - const workflowId = plan.id; +export type RTEOptions = { + resolvers?: LazyResolvers; + logger?: Logger; + workerPath?: string; // TODO maybe the public API doesn't expose this + repoDir?: string; + noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? - // Pull options out of the plan so that all options are in one place - const { options, ...plan } = executionPlan; + autoinstall: AutoinstallOptions; +}; - // initial state for this workflow run - // TODO let's create a util function for this (nice for testing) - const state = createState(plan); - // the engine does need to be able to report on the state of each workflow - api.registerWorkflow(state); +// Create the engine and handle user-facing stuff, like options parsing +// and defaulting +const createAPI = function (serverId?: string, options: RTEOptions = {}) { + let { repoDir } = options; + + const logger = options.logger || createLogger('RTE', { level: 'debug' }); + + if (!repoDir) { + if (process.env.OPENFN_RTE_REPO_DIR) { + repoDir = process.env.OPENFN_RTE_REPO_DIR; + } else { + repoDir = '/tmp/openfn/repo'; + logger.warn('Using default repodir'); + logger.warn( + 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' + ); + } + } - const events = createWorkflowEvents(api); + logger.info('repoDir set to ', repoDir); - listeners[workflowId] = events; + // TODO I dunno, does the engine have an id? + // I think that's a worker concern, especially + // as there's a 1:1 worker:engine mapping + // const id = serverId || crypto.randomUUID(); - // this context API thing is the internal api / engine - // each has a bespoke event emitter but otherwise a common interface - const contextAPI: EngineAPI = { - ...api, - ...events, - }; + // TODO we want to get right down to this - execute(contextAPI, state, options); + // Create the internal API + const engine = createApi(options); - // return the event emitter (not the full engine API though) - return events; + // Return the external API + return { + execute: engine.execute, + listen: engine.listen, }; - - // // how will this actually work? - // api.listen = ( - // attemptId: string, - // listeners: Record - // ) => { - // // const handler = (eventName) => { - // // if () - // // } - // const events = listeners[workflowId]; - // for (const evt of listeners) { - // events.on(evt, listeners[evt]); - // } - - // // TODO return unsubscribe handle - // }; - - // we can have global reporting like this - api.getStatus = (workflowId) => state.workflows[workflowId].status; - - initWorkers(api); - - return api; }; export default createAPI; diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index c62aa97e0..79df475cd 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -11,8 +11,9 @@ type WorkerEvent = { // Adds a `callWorker` function to the API object, which will execute a task in a worker export default function initWorkers(api: EngineAPI, workerPath: string) { + // TODO can we verify the worker path and throw if it's invalid? + // workerpool won't complain if we give it a nonsense path const workers = createWorkers(workerPath); - api.callWorker = (task: string, args: any[] = [], events: any = {}) => workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { diff --git a/packages/engine-multi/src/api/compile.ts b/packages/engine-multi/src/api/compile.ts index 1303f3f39..6da397cc4 100644 --- a/packages/engine-multi/src/api/compile.ts +++ b/packages/engine-multi/src/api/compile.ts @@ -6,7 +6,7 @@ import type { Logger } from '@openfn/logger'; import compile, { preloadAdaptorExports } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; import { EngineAPI, WorkflowState } from '../types'; -import { RTEOptions } from '../engine'; +import { RTEOptions } from '../__engine'; // TODO this compiler is going to change anyway to run just in time // the runtime will have an onCompile hook diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 54c2f3caa..6bc822285 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -47,7 +47,7 @@ export const workflowComplete = ( state: WorkflowState, // mutable workflow state event: WorkerCompletePayload // the event published by the runtime itself ({ workflowId, threadId }) ) => { - const { workflowId, state: result } = event; + const { workflowId, state: result, threadId } = event; api.logger.success('complete workflow ', workflowId); api.logger.info(state); @@ -68,6 +68,7 @@ export const workflowComplete = ( // forward the event on to any external listeners api.emit(WORKFLOW_COMPLETE, { workflowId: workflowId, + threadId, duration: state.duration, state: result, }); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 6f5fe0e0c..431255c95 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -1,207 +1,171 @@ -import path from 'node:path'; -import crypto from 'node:crypto'; -import { fileURLToPath } from 'url'; import { EventEmitter } from 'node:events'; -import workerpool from 'workerpool'; -import { ExecutionPlan } from '@openfn/runtime'; -import createLogger, { JSONLog, Logger } from '@openfn/logger'; - -import * as e from './events'; -import compile from './api/compile'; -import execute from './runners/execute'; -import autoinstall, { AutoinstallOptions } from './api/autoinstall'; - -export type State = any; // TODO I want a nice state def with generics - -// Archive of every workflow we've run -// Fine to just keep in memory for now -type WorkflowStats = { - id: string; - name?: string; // TODO what is name? this is irrelevant? - status: 'pending' | 'done' | 'err'; - startTime?: number; - threadId?: number; - duration?: number; - error?: string; - result?: any; // State - plan: ExecutionPlan; -}; - -type Resolver = (id: string) => Promise; - -// A list of helper functions which basically resolve ids into JSON -// to lazy load assets -export type LazyResolvers = { - credentials?: Resolver; - state?: Resolver; - expressions?: Resolver; -}; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { ExecutionPlan } from '@openfn/runtime'; + +import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from './events'; +import initWorkers from './api/call-worker'; +import createState from './api/create-state'; +import execute from './api/execute'; + +import type { RTEOptions } from './api'; +import type { + EngineAPI, + EngineEvents, + EventHandler, + WorkflowState, +} from './types'; + +// For each workflow, create an API object with its own event emitter +// this is a bt wierd - what if the emitter went on state instead? +const createWorkflowEvents = (api: EngineAPI, workflowId: string) => { + //create a bespoke event emitter + const events = new EventEmitter(); -export type RTEOptions = { - resolvers?: LazyResolvers; - logger?: Logger; - workerPath?: string; - repoDir?: string; - noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? + // proxy all events to the main emitter + // uh actually there may be no point in this + function proxy(event: string) { + events.on(event, (evt) => { + // ensure the attempt id is on the event + evt.workflowId = workflowId; + const newEvt = { + ...evt, + workflowId: workflowId, + }; + + api.emit(event, newEvt); + }); + } + proxy(WORKFLOW_START); + proxy(WORKFLOW_COMPLETE); + // proxy(JOB_START); + // proxy(JOB_COMPLETE); + proxy(WORKFLOW_LOG); - autoinstall: AutoinstallOptions; + return events; }; -const createRTM = function (serverId?: string, options: RTEOptions = {}) { - const { noCompile } = options; - let { repoDir, workerPath } = options; - - const id = serverId || crypto.randomUUID(); - const logger = options.logger || createLogger('RTM', { level: 'debug' }); - - const allWorkflows: Map = new Map(); - const activeWorkflows: string[] = []; - - // TODO we want to get right down to this - - // Create the internal API - const api = createApi(); - - // Return the external API - return { - execute: api.execute, - listen: api.listen, +// TODO this is a quick and dirty to get my own claass name in the console +// (rather than EventEmitter) +// But I should probably lean in to the class more for typing and stuff +class Engine extends EventEmitter {} + +// TODO this is actually the api that each execution gets +// its nice to separate that from the engine a bit +class ExecutionContext extends EventEmitter { + constructor(workflowState, logger, callWorker) { + super(); + this.logger = logger; + this.callWorker = callWorker; + this.state = workflowState; + } +} + +// This creates the internal API +// tbh this is actually the engine, right, this is where stuff happens +// the api file is more about the public api I think +const createEngine = (options: RTEOptions, workerPath?: string) => { + // internal state + const allWorkflows: Map = new Map(); + const listeners = {}; + // TODO I think this is for later + //const activeWorkflows: string[] = []; + + let resolvedWorkerPath; + if (workerPath) { + // If a path to the worker has been passed in, just use it verbatim + // We use this to pass a mock worker for testing purposes + resolvedWorkerPath = workerPath; + } else { + // By default, we load ./worker.js but can't rely on the working dir to find it + const dirname = path.dirname(fileURLToPath(import.meta.url)); + resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + } + options.logger.debug('Loading workers from ', resolvedWorkerPath); + + const engine = new Engine() as EngineAPI; + + initWorkers(engine, resolvedWorkerPath); + + // TODO I think this needs to be like: + // take a plan + // create, register and return a state object + // should it also load the initial data clip? + // when does that happen? No, that's inside execute + const registerWorkflow = (plan: ExecutionPlan) => { + // TODO throw if already registered? + const state = createState(plan); + allWorkflows[state.id] = state; + return state; }; - // let resolvedWorkerPath; - // if (workerPath) { - // // If a path to the worker has been passed in, just use it verbatim - // // We use this to pass a mock worker for testing purposes - // resolvedWorkerPath = workerPath; - // } else { - // // By default, we load ./worker.js but can't rely on the working dir to find it - // const dirname = path.dirname(fileURLToPath(import.meta.url)); - // resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); - // } - // const workers = workerpool.pool(resolvedWorkerPath); - - const events = new EventEmitter(); - - if (!repoDir) { - if (process.env.OPENFN_RTE_REPO_DIR) { - repoDir = process.env.OPENFN_RTE_REPO_DIR; - } else { - repoDir = '/tmp/openfn/repo'; - logger.warn('Using default repodir'); - logger.warn( - 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' - ); - } - } - logger.info('repoDir set to ', repoDir); - - // const onWorkflowStarted = (workflowId: string, threadId: number) => { - // logger.info('starting workflow ', workflowId); - // const workflow = allWorkflows.get(workflowId)!; - - // if (workflow.startTime) { - // // TODO this shouldn't throw.. but what do we do? - // // We shouldn't run a workflow that's been run - // // Every workflow should have a unique id - // // maybe the RTM doesn't care about this - // throw new Error(`Workflow with id ${workflowId} is already started`); - // } - // workflow.startTime = new Date().getTime(); - // workflow.duration = -1; - // workflow.threadId = threadId; - // activeWorkflows.push(workflowId); - - // // forward the event on to any external listeners - // events.emit(e.WORKFLOW_START, { - // workflowId, - // // Should we publish anything else here? - // }); - // }; - - // const completeWorkflow = (workflowId: string, state: any) => { - // logger.success('complete workflow ', workflowId); - // logger.info(state); - // if (!allWorkflows.has(workflowId)) { - // throw new Error(`Workflow with id ${workflowId} is not defined`); - // } - // const workflow = allWorkflows.get(workflowId)!; - // workflow.status = 'done'; - // workflow.result = state; - // workflow.duration = new Date().getTime() - workflow.startTime!; - // const idx = activeWorkflows.findIndex((id) => id === workflowId); - // activeWorkflows.splice(idx, 1); - - // // forward the event on to any external listeners - // events.emit(e.WORKFLOW_COMPLETE, { - // id: workflowId, - // duration: workflow.duration, - // state, - // }); - // }; - - // // Catch a log coming out of a job within a workflow - // // Includes runtime logging (is this right?) - // const onWorkflowLog = (workflowId: string, message: JSONLog) => { - // // Seamlessly proxy the log to the local stdout - // // TODO runtime logging probably needs to be at info level? - // // Debug information is mostly irrelevant for lightning - // const newMessage = { - // ...message, - // // Prefix the job id in all local jobs - // // I'm sure there are nicer, more elegant ways of doing this - // message: [`[${workflowId}]`, ...message.message], - // }; - // logger.proxy(newMessage); - // events.emit(e.WORKFLOW_LOG, { - // workflowId, - // message, - // }); - // }; - - // How much of this happens inside the worker? - // Shoud the main thread handle compilation? Has to if we want to cache - // Unless we create a dedicated compiler worker - // TODO error handling, timeout - const handleExecute = async (plan: ExecutionPlan) => { - const options = { - repoDir, - }; - const context = { plan, logger, workers, options /* api */ }; - - logger.debug('Executing workflow ', plan.id); - - allWorkflows.set(plan.id!, { - id: plan.id!, - status: 'pending', - plan, + const getWorkflowState = (workflowId: string) => allWorkflows[workflowId]; + + const getWorkflowStatus = (workflowId: string) => + allWorkflows[workflowId]?.status; + + // TODO are we totally sure this takes a standard xplan? + const executeWrapper = (plan: ExecutionPlan) => { + const state = registerWorkflow(plan); + + const events = createWorkflowEvents(engine, plan.id); + listeners[plan.id] = events; + + // this context API thing is the internal api / engine + // each has a bespoke event emitter but otherwise a common interface + // const api: EngineAPI = { + // ...engine, + // ...events, + // }; + // yeah this feels nasty + // also in debugging the name will be wrong + // i think maybe we just do new Engine(state) + // and that creates an API with shared state + // also this internal engine is a bit different + // i think it's just logger and events? + // so I'm back to it being a context. Interesting. + // Ok so now we have an executioncontext, which I'll create soon + // maybe it should just have state on it + const api = Object.assign(events, { + // workerPath: resolvedWorkerPath, + logger: options.logger, + callWorker: engine.callWorker, + // registerWorkflow, + // getWorkflowState, + // getWorkflowStatus, + // execute: executeWrapper, + // listen, }); - const adaptorPaths = await autoinstall(context); + execute(api, state, options); - if (!noCompile) { - context.plan = await compile(context); - } + // return the event emitter + return events; + }; - logger.debug('workflow compiled ', plan.id); - const result = await execute(context, adaptorPaths, { - start: onWorkflowStarted, - log: onWorkflowLog, - }); - completeWorkflow(plan.id!, result); + const listen = ( + workflowId: string, + handlers: Record + ) => { + const events = listeners[workflowId]; + for (const evt in handlers) { + events.on(evt, handlers[evt]); + } - logger.debug('finished executing workflow ', plan.id); - // Return the result - // Note that the mock doesn't behave like ths - // And tbf I don't think we should keep the promise open - there's no point? - return result; + // TODO return unsubscribe handle }; - return { - id, - on: (type: string, fn: (...args: any[]) => void) => events.on(type, fn), - once: (type: string, fn: (...args: any[]) => void) => events.once(type, fn), - execute: handleExecute, - }; + engine.emit('test'); + + return Object.assign(engine, { + workerPath: resolvedWorkerPath, + logger: options.logger, + registerWorkflow, + getWorkflowState, + getWorkflowStatus, + execute: executeWrapper, + listen, + }); }; -export default createRTM; +export default createEngine; diff --git a/packages/engine-multi/src/index.ts b/packages/engine-multi/src/index.ts index e4f27252e..a85794d60 100644 --- a/packages/engine-multi/src/index.ts +++ b/packages/engine-multi/src/index.ts @@ -1,3 +1,3 @@ -import createEngine from './engine'; +import createAPI from './api'; -export default createEngine; +export default createAPI; diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index 6e6cd0070..8393c6f4c 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -1,8 +1,9 @@ import test from 'ava'; -import createAPI from '../src/api'; +import createAPI from '../src/__api'; -// thes are tests on the api functions generally +// thes are tests on the public api functions generally +// so these are very high level tests // no need to test the event stuff - startworkflow etc // maybe we can check the keys exist, although we'll quickly know if we dont diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 9c0f86d84..2cd5c46be 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -3,8 +3,11 @@ import path from 'node:path'; import { createMockLogger } from '@openfn/logger'; import { createPlan } from './util'; -import Manager from '../src/engine'; +import createEngine from '../src/engine'; import * as e from '../src/events'; +import { ExecutionPlan } from '@openfn/runtime'; + +// TOOD this becomes low level tests on the internal engine api const logger = createMockLogger('', { level: 'debug' }); @@ -22,96 +25,136 @@ test.afterEach(() => { logger._reset(); }); -test('Should create a new manager', (t) => { - const rtm = Manager('x', options); - t.truthy(rtm); - t.truthy(rtm.execute); - t.truthy(rtm.on); - t.truthy(rtm.once); - t.is(rtm.id, 'x'); +test('create an engine', (t) => { + const engine = createEngine({ logger }); + t.truthy(engine); + t.is(engine.constructor.name, 'Engine'); + t.truthy(engine.execute); + t.truthy(engine.on); + t.truthy(engine.once); + t.truthy(engine.emit); }); -test('Should run a mock job with a simple return value', async (t) => { - const state = { data: { x: 1 } }; - const rtm = Manager('x', options); - const plan = createPlan({ - expression: `() => (${JSON.stringify(state)})`, - }); - const result = await rtm.execute(plan); - t.deepEqual(result, state); -}); +test('register a workflow', (t) => { + const plan = { id: 'z' }; + const engine = createEngine({ logger }); -test('Should not explode if no adaptor is passed', async (t) => { - const state = { data: { x: 1 } }; - const rtm = Manager('x', options); - const plan = createPlan({ - expression: `() => (${JSON.stringify(state)})`, - }); + const state = engine.registerWorkflow(plan); - // @ts-ignore - delete plan.jobs[0].adaptor; - const result = await rtm.execute(plan); - t.deepEqual(result, state); + t.is(state.status, 'pending'); + t.is(state.id, plan.id); + t.deepEqual(state.plan, plan); }); -test('events: workflow-start', async (t) => { - const rtm = Manager('x', options); +test('get workflow state', (t) => { + const plan = { id: 'z' } as ExecutionPlan; + const engine = createEngine({ logger }); - let id; - let didCall; - rtm.on(e.WORKFLOW_START, ({ workflowId }) => { - didCall = true; - id = workflowId; - }); + const s = engine.registerWorkflow(plan); - const plan = createPlan(); - await rtm.execute(plan); + const state = engine.getWorkflowState(plan.id); - t.true(didCall); - t.is(id, plan.id); + t.deepEqual(state, s); }); -test('events: workflow-complete', async (t) => { - const rtm = Manager('x', options); - - let didCall; - let evt; - rtm.on(e.WORKFLOW_COMPLETE, (e) => { - didCall = true; - evt = e; - }); +test('use the default worker path', (t) => { + const engine = createEngine({ logger }); + // this is how the path comes out in the test framework + t.true(engine.workerPath.endsWith('src/worker.js')); +}); - const plan = createPlan(); - await rtm.execute(plan); +// Note that even though this is a nonsense path, we get no error at this point +test('use a custom worker path', (t) => { + const p = 'jam'; + const engine = createEngine({ logger }, p); + // this is how the path comes out in the test framework + t.is(engine.workerPath, p); +}); - t.true(didCall); - t.is(evt.id, plan.id); - t.assert(!isNaN(evt.duration)); - t.deepEqual(evt.state, { data: { answer: 42 } }); +test('execute with test worker and trigger workflow-complete', (t) => { + return new Promise((done) => { + const p = path.resolve('test/worker-functions.js'); + const engine = createEngine( + { + logger, + noCompile: true, + autoinstall: { + handleIsInstalled: async () => true, + }, + }, + p + ); + + const plan = { + id: 'a', + jobs: [ + { + expression: '22', + }, + ], + }; + + engine.execute(plan).on(e.WORKFLOW_COMPLETE, ({ state, threadId }) => { + t.is(state, 22); + t.truthy(threadId); // proves (sort of) that this has run in a worker + done(); + }); + }); }); -// TODO: workflow log should also include runtime events, which maybe should be reflected here +test.only('listen to workflow-complete', (t) => { + return new Promise((done) => { + const p = path.resolve('test/worker-functions.js'); + const engine = createEngine( + { + logger, + noCompile: true, + autoinstall: { + handleIsInstalled: async () => true, + }, + }, + p + ); + + const plan = { + id: 'a', + jobs: [ + { + expression: '33', + }, + ], + }; + + engine.execute(plan); + engine.listen(plan.id, { + [e.WORKFLOW_COMPLETE]: ({ state, threadId }) => { + t.is(state, 33); + t.truthy(threadId); // proves (sort of) that this has run in a worker + done(); + }, + }); + }); +}); -test('events: workflow-log from a job', async (t) => { +test.skip('Should run a mock job with a simple return value', async (t) => { + const state = { data: { x: 1 } }; const rtm = Manager('x', options); - - let didCall; - let evt; - rtm.on(e.WORKFLOW_LOG, (e) => { - didCall = true; - evt = e; + const plan = createPlan({ + expression: `() => (${JSON.stringify(state)})`, }); + const result = await rtm.execute(plan); + t.deepEqual(result, state); +}); +test.skip('Should not explode if no adaptor is passed', async (t) => { + const state = { data: { x: 1 } }; + const rtm = Manager('x', options); const plan = createPlan({ - expression: `(s) => { - console.log('log me') - return s; - }`, + expression: `() => (${JSON.stringify(state)})`, }); - await rtm.execute(plan); - t.true(didCall); - t.is(evt.message.level, 'info'); - t.deepEqual(evt.message.message, ['log me']); - t.is(evt.message.name, 'JOB'); + // @ts-ignore + delete plan.jobs[0].adaptor; + const result = await rtm.execute(plan); + t.deepEqual(result, state); }); diff --git a/packages/engine-multi/test/worker-functions.js b/packages/engine-multi/test/worker-functions.js index 028690413..c4e1221e3 100644 --- a/packages/engine-multi/test/worker-functions.js +++ b/packages/engine-multi/test/worker-functions.js @@ -14,35 +14,34 @@ workerpool.worker({ return result; }, - // // very basic simulation of a run - // hmm, don't use this, use mock worker instead - // run: (plan, adaptorPaths) => { - // workerpool.workerEmit({ type: e.WORKFLOW_START, workflowId, threadId }); - // try { - // // TODO - // // workerpool.workerEmit({ - // // type: e.WORKFLOW_LOG, - // // }); - - // // or something - // const result = eval(plan); - - // workerpool.workerEmit({ - // type: e.WORKFLOW_COMPLETE, - // workflowId, - // state: result, - // }); + // very very simple intepretation of a run function + // Most tests should use the mock-worker instead + run: (plan, adaptorPaths) => { + const workflowId = plan.id; + workerpool.workerEmit({ + type: 'workflow-start', + workflowId, + threadId, + }); + try { + const [job] = plan.jobs; + const result = eval(job.expression); - // // For tests - // return result; - // } catch (err) { - // console.error(err); - // // @ts-ignore TODO sort out error typing - // workerpool.workerEmit({ - // type: e.WORKFLOW_ERROR, - // workflowId, - // message: err.message, - // }); - // } - // }, + workerpool.workerEmit({ + type: 'workflow-complete', + workflowId, + state: result, + threadId, + }); + } catch (err) { + console.error(err); + // @ts-ignore TODO sort out error typing + workerpool.workerEmit({ + type: 'workflow-error', + workflowId, + message: err.message, + threadId, + }); + } + }, }); From d7c71889b0bf7c3f7fa620310dd3d745b2da9e01 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 11 Oct 2023 16:36:27 +0100 Subject: [PATCH 134/232] engine: use a context object I think I've finally worked out hte api/context/state stuff - now it's just a context object, itself an event emitter, which tracks state --- packages/engine-multi/src/api/autoinstall.ts | 26 +++--- packages/engine-multi/src/api/compile.ts | 18 ++--- packages/engine-multi/src/api/execute.ts | 32 +++----- packages/engine-multi/src/api/lifecycle.ts | 28 +++---- packages/engine-multi/src/engine.ts | 75 ++++++++---------- packages/engine-multi/src/types.d.ts | 10 ++- packages/engine-multi/test/api.test.ts | 8 +- .../engine-multi/test/api/autoinstall.test.ts | 79 +++++++++---------- .../engine-multi/test/api/execute.test.ts | 69 ++++++++-------- .../engine-multi/test/api/lifecycle.test.ts | 50 ++++++------ packages/engine-multi/test/engine.test.ts | 2 +- 11 files changed, 184 insertions(+), 213 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index ac8a6e6e0..026145093 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -9,11 +9,10 @@ import { } from '@openfn/runtime'; import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; -import { EngineAPI, WorkflowState } from '../types'; +import { EngineAPI, ExecutionContext, WorkflowState } from '../types'; // none of these options should be on the plan actually export type AutoinstallOptions = { - repoDir?: string; skipRepoValidation?: boolean; handleInstall?(fn: string, repoDir: string, logger: Logger): Promise; handleIsInstalled?( @@ -25,25 +24,22 @@ export type AutoinstallOptions = { const pending: Record> = {}; -const autoinstall = async ( - api: EngineAPI, - state: WorkflowState, - options: AutoinstallOptions -): Promise => { - const { logger } = api; +const autoinstall = async (context: ExecutionContext): Promise => { + const { logger, state, options } = context; const { plan } = state; const { repoDir } = options; + const autoinstallOptions = options.autoinstall || {}; - const installFn = options.handleInstall || install; - const isInstalledFn = options.handleIsInstalled || isInstalled; + const installFn = autoinstallOptions?.handleInstall || install; + const isInstalledFn = autoinstallOptions?.handleIsInstalled || isInstalled; let didValidateRepo = false; - const { skipRepoValidation } = options; + const { skipRepoValidation } = autoinstallOptions; - if (!skipRepoValidation && !didValidateRepo && options.repoDir) { + if (!skipRepoValidation && !didValidateRepo && repoDir) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate - await ensureRepo(options.repoDir, logger); + await ensureRepo(repoDir, logger); didValidateRepo = true; } @@ -62,7 +58,9 @@ const autoinstall = async ( if (needsInstalling) { if (!pending[a]) { // add a promise to the pending array - pending[a] = installFn(a, options.repoDir, logger); + pending[a] = installFn(a, repoDir, logger).then(() => { + delete pending[a]; + }); } // Return the pending promise (safe to do this multiple times) await pending[a].then(); diff --git a/packages/engine-multi/src/api/compile.ts b/packages/engine-multi/src/api/compile.ts index 6da397cc4..469e8af1c 100644 --- a/packages/engine-multi/src/api/compile.ts +++ b/packages/engine-multi/src/api/compile.ts @@ -5,23 +5,17 @@ import type { Logger } from '@openfn/logger'; import compile, { preloadAdaptorExports } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; -import { EngineAPI, WorkflowState } from '../types'; -import { RTEOptions } from '../__engine'; +import { ExecutionContext } from '../types'; // TODO this compiler is going to change anyway to run just in time // the runtime will have an onCompile hook // We'll keep this for now though while we get everything else working -export default async ( - api: EngineAPI, - state: WorkflowState, - options: Pick = {} -) => { - const { logger } = api; - const { plan } = state; +export default async (context: ExecutionContext) => { + const { logger, state, options } = context; const { repoDir, noCompile } = options; - if (!noCompile) { - for (const job of plan.jobs) { + if (!noCompile && state.plan?.jobs?.length) { + for (const job of state.plan.jobs) { if (job.expression) { job.expression = await compileJob( job.expression as string, @@ -32,8 +26,6 @@ export default async ( } } } - - return plan; }; // TODO copied out of CLI - how can we share code here? diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index f39f77d14..553bdb229 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -1,8 +1,6 @@ -// this replaces the runner - // Execute a compiled workflow import * as e from '../events'; -import { EngineAPI, WorkflowState } from '../types'; +import { ExecutionContext } from '../types'; import autoinstall from './autoinstall'; import compile from './compile'; @@ -12,32 +10,28 @@ import { workflowStart, workflowComplete, log } from './lifecycle'; // Is it better to just return the handler? // But then this function really isn't doing so much // (I guess that's true anyway) -const execute = async ( - api: EngineAPI, - state: WorkflowState, - options: RTEOptions -) => { - const adaptorPaths = await autoinstall(api, state, options.autoinstall); - await compile(api, state, options); +const execute = async (context: ExecutionContext) => { + const { state, callWorker, logger } = context; + + const adaptorPaths = await autoinstall(context); + await compile(context); const events = { [e.WORKFLOW_START]: (evt) => { - workflowStart(api, state, evt); + workflowStart(context, evt); }, [e.WORKFLOW_COMPLETE]: (evt) => { - workflowComplete(api, state, evt); + workflowComplete(context, evt); }, [e.WORKFLOW_LOG]: (evt) => { - log(api, state, evt); + log(context, evt); }, }; - return api - .callWorker('run', [state.plan, adaptorPaths], events) - .catch((e) => { - // TODO what about errors then? - api.logger.error(e); - }); + return callWorker('run', [state.plan, adaptorPaths], events).catch((e) => { + // TODO what about errors then? + logger.error(e); + }); }; export default execute; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 6bc822285..6219e527b 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -1,6 +1,7 @@ import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from '../events'; import { EngineAPI, + ExecutionContext, WorkerCompletePayload, WorkerLogPayload, WorkerStartPayload, @@ -8,13 +9,13 @@ import { } from '../types'; export const workflowStart = ( - api: EngineAPI, // general API - state: WorkflowState, // mutable workflow state + context: ExecutionContext, event: WorkerStartPayload // the event published by the runtime itself ({ workflowId, threadId }) ) => { + const { state, logger } = context; const { workflowId, threadId } = event; - api.logger.info('starting workflow ', workflowId); + logger.info('starting workflow ', workflowId); // where would this throw get caught? if (state.startTime) { @@ -36,21 +37,21 @@ export const workflowStart = ( // api.activeWorkflows.push(workflowId); // forward the event on to any external listeners - api.emit(WORKFLOW_START, { + context.emit(WORKFLOW_START, { workflowId, // if this is a bespoke emitter it can be implied, which is nice // Should we publish anything else here? }); }; export const workflowComplete = ( - api: EngineAPI, // general API - state: WorkflowState, // mutable workflow state + context: ExecutionContext, event: WorkerCompletePayload // the event published by the runtime itself ({ workflowId, threadId }) ) => { + const { logger, state } = context; const { workflowId, state: result, threadId } = event; - api.logger.success('complete workflow ', workflowId); - api.logger.info(state); + logger.success('complete workflow ', workflowId); + logger.info(state); // TODO I don't know how we'd get here in this architecture // if (!allWorkflows.has(workflowId)) { @@ -66,7 +67,7 @@ export const workflowComplete = ( // activeWorkflows.splice(idx, 1); // forward the event on to any external listeners - api.emit(WORKFLOW_COMPLETE, { + context.emit(WORKFLOW_COMPLETE, { workflowId: workflowId, threadId, duration: state.duration, @@ -75,11 +76,10 @@ export const workflowComplete = ( }; export const log = ( - api: EngineAPI, // general API - state: WorkflowState, // mutable workflow state + context: ExecutionContext, event: WorkerLogPayload // the event published by the runtime itself ({ workflowId, threadId }) ) => { - const { id } = state; + const { id } = context.state; // // TODO not sure about this stuff, I think we can drop it? // const newMessage = { // ...message, @@ -87,9 +87,9 @@ export const log = ( // // I'm sure there are nicer, more elegant ways of doing this // message: [`[${workflowId}]`, ...message.message], // }; - api.logger.proxy(event.message); + context.logger.proxy(event.message); - api.emit(WORKFLOW_LOG, { + context.emit(WORKFLOW_LOG, { workflowId: id, ...event.message, }); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 431255c95..2baff1b40 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -18,14 +18,15 @@ import type { // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? -const createWorkflowEvents = (api: EngineAPI, workflowId: string) => { - //create a bespoke event emitter - const events = new EventEmitter(); - +const createWorkflowEvents = ( + engine: EngineAPI, + context: ExecutionContext, + workflowId: string +) => { // proxy all events to the main emitter // uh actually there may be no point in this function proxy(event: string) { - events.on(event, (evt) => { + context.on(event, (evt) => { // ensure the attempt id is on the event evt.workflowId = workflowId; const newEvt = { @@ -33,7 +34,7 @@ const createWorkflowEvents = (api: EngineAPI, workflowId: string) => { workflowId: workflowId, }; - api.emit(event, newEvt); + engine.emit(event, newEvt); }); } proxy(WORKFLOW_START); @@ -42,7 +43,7 @@ const createWorkflowEvents = (api: EngineAPI, workflowId: string) => { // proxy(JOB_COMPLETE); proxy(WORKFLOW_LOG); - return events; + return context; }; // TODO this is a quick and dirty to get my own claass name in the console @@ -53,11 +54,12 @@ class Engine extends EventEmitter {} // TODO this is actually the api that each execution gets // its nice to separate that from the engine a bit class ExecutionContext extends EventEmitter { - constructor(workflowState, logger, callWorker) { + constructor(workflowState, logger, callWorker, options) { super(); this.logger = logger; this.callWorker = callWorker; this.state = workflowState; + this.options = options; } } @@ -65,8 +67,10 @@ class ExecutionContext extends EventEmitter { // tbh this is actually the engine, right, this is where stuff happens // the api file is more about the public api I think const createEngine = (options: RTEOptions, workerPath?: string) => { - // internal state + // TODO this is probably states now, although contexts sort of subsumes it const allWorkflows: Map = new Map(); + + // TODO this contexts now const listeners = {}; // TODO I think this is for later //const activeWorkflows: string[] = []; @@ -108,39 +112,26 @@ const createEngine = (options: RTEOptions, workerPath?: string) => { const executeWrapper = (plan: ExecutionPlan) => { const state = registerWorkflow(plan); - const events = createWorkflowEvents(engine, plan.id); - listeners[plan.id] = events; - - // this context API thing is the internal api / engine - // each has a bespoke event emitter but otherwise a common interface - // const api: EngineAPI = { - // ...engine, - // ...events, - // }; - // yeah this feels nasty - // also in debugging the name will be wrong - // i think maybe we just do new Engine(state) - // and that creates an API with shared state - // also this internal engine is a bit different - // i think it's just logger and events? - // so I'm back to it being a context. Interesting. - // Ok so now we have an executioncontext, which I'll create soon - // maybe it should just have state on it - const api = Object.assign(events, { - // workerPath: resolvedWorkerPath, - logger: options.logger, - callWorker: engine.callWorker, - // registerWorkflow, - // getWorkflowState, - // getWorkflowStatus, - // execute: executeWrapper, - // listen, - }); - - execute(api, state, options); - - // return the event emitter - return events; + const context = new ExecutionContext( + state, + options.logger, + engine.callWorker, + options + ); + + listeners[plan.id] = createWorkflowEvents(engine, context, engine, plan.id); + + execute(context); + + // hmm. Am I happy to pass the internal workflow state OUT of the handler? + // I'd rather have like a proxy emitter or something + // also I really only want passive event handlers, I don't want interference from outside + return { + on: (...args) => context.on(...args), + once: context.once, + off: context.off, + }; + // return context; }; const listen = ( diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index 5e599a56e..d5104d9ec 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -3,6 +3,7 @@ import { JSONLog, Logger } from '@openfn/logger'; import { ExecutionPlan } from '@openfn/runtime'; import type { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; +import { RTEOptions } from './api'; // These are the external events published but he api and listen type WorkflowStartEvent = 'workflow-start'; @@ -76,6 +77,7 @@ export type WorkerStartPayload = { export type WORKER_COMPLETE = 'worker-complete'; export type WorkerCompletePayload = { + threadId: string; workflowId: string; state: any; }; @@ -83,6 +85,7 @@ export type WorkerCompletePayload = { // TODO confusion over this and events.ts export type WORKER_LOG = 'worker-log'; export type WorkerLogPayload = { + threadId: string; workflowId: string; message: JSONLog; }; @@ -117,9 +120,10 @@ export type WorkflowState = { }; // this is the internal engine API -export interface EngineAPI extends EventEmitter { +export interface ExecutionContext extends EventEmitter { + options: RTEOptions; // TODO maybe. bring them in here? + state: WorkflowState; logger: Logger; - callWorker: ( task: string, args: any[] = [], @@ -127,6 +131,8 @@ export interface EngineAPI extends EventEmitter { ) => workerpool.Promise; } +export interface EngineAPI extends EventEmitter {} + interface RuntimeEngine extends EventEmitter { //id: string // human readable instance id // actually I think the id is on the worker, not the engine diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index 8393c6f4c..e0430208d 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -14,8 +14,8 @@ test.todo('execute should proxy events'); test.todo('listen'); test.todo('log'); -test('callWorker', (t) => { - const api = createAPI(); +// test('callWorker', (t) => { +// const api = createAPI(); - t.pass(); -}); +// t.pass(); +// }); diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index d9685cea7..e48583977 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -2,7 +2,7 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; -import { EngineAPI, WorkflowState } from '../../src/types'; +import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; type PackageJson = { name: string; @@ -14,16 +14,26 @@ const mockIsInstalled = (pkg: PackageJson) => async (specifier: string) => { return pkg.dependencies.hasOwnProperty(alias); }; -// TODO should this write to package json? -// I don't think there's any need const mockHandleInstall = async (specifier: string): Promise => new Promise((r) => r()).then(); -const mockLogger = createMockLogger(); +const logger = createMockLogger(); -const api = { - logger: mockLogger, -} as unknown as EngineAPI; +const createContext = (autoinstallOpts?, jobs?: any[]) => + ({ + logger, + state: { + plan: { + jobs: jobs || [{ adaptor: 'x@1.0.0' }], + }, + }, + options: { + autoinstall: autoinstallOpts || { + handleInstall: mockHandleInstall, + handleIsInstalled: mockIsInstalled, + }, + }, + } as unknown as ExecutionContext); test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ @@ -88,18 +98,13 @@ test.serial('autoinstall: should call both mock functions', async (t) => { return; }; - const state = { - plan: { - jobs: [{ adaptor: 'x@1.0.0' }], - }, - } as WorkflowState; - - const options = { + const autoinstallOpts = { handleInstall: mockInstall, handleIsInstalled: mockIsInstalled, }; + const context = createContext(autoinstallOpts); - await autoinstall(api, state, options); + await autoinstall(context); t.true(didCallIsInstalled); t.true(didCallInstall); @@ -117,48 +122,40 @@ test.serial( }); const options = { + skipRepoValidation: true, handleInstall: mockInstall, handleIsInstalled: async () => false, }; - const state = { - plan: { - jobs: [{ adaptor: 'z@1.0.0' }], - }, - } as WorkflowState; + const context = createContext(options); - await Promise.all([ - autoinstall(api, state, options), - autoinstall(api, state, options), - ]); + await Promise.all([autoinstall(context), autoinstall(context)]); t.is(callCount, 1); } ); test.serial('autoinstall: return a map to modules', async (t) => { - const state = { - plan: { - // Note that we have difficulty now if a workflow imports two versions of the same adaptor - jobs: [ - { - adaptor: 'common@1.0.0', - }, - { - adaptor: 'http@1.0.0', - }, - ], + const jobs = [ + { + adaptor: 'common@1.0.0', }, - } as WorkflowState; + { + adaptor: 'http@1.0.0', + }, + ]; - const options = { + const context = createContext(null, jobs); + context.options = { repoDir: 'a/b/c', - skipRepoValidation: true, - handleInstall: async () => true, - handleIsInstalled: async () => false, + autoinstall: { + skipRepoValidation: true, + handleInstall: async () => {}, + handleIsInstalled: async () => false, + }, }; - const result = await autoinstall(api, state, options); + const result = await autoinstall(context); t.deepEqual(result, { common: { path: 'a/b/c/node_modules/common_1.0.0' }, diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index f10ffda79..204537ef1 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import test from 'ava'; import { EventEmitter } from 'node:events'; -import { EngineAPI } from '../../src/types'; +import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; import initWorkers from '../../src/api/call-worker'; import execute from '../../src/api/execute'; import { createMockLogger } from '@openfn/logger'; @@ -10,18 +10,21 @@ import { WORKFLOW_LOG, WORKFLOW_START, } from '../../src/events'; +import { RTEOptions } from '../../src/api'; const workerPath = path.resolve('dist/mock-worker.js'); -// Mock API object -let api; - -test.before(() => { - api = new EventEmitter() as EngineAPI; - api.logger = createMockLogger(); - +const createContext = ({ state, options }: Partial = {}) => { + const api = new EventEmitter(); + Object.assign(api, { + logger: createMockLogger(), + state: state || {}, + options, + // logger: not used + }) as unknown as ExecutionContext; initWorkers(api, workerPath); -}); + return api; +}; const plan = { id: 'x', @@ -36,39 +39,34 @@ const plan = { const options = { noCompile: true, autoinstall: { - handleInstall: async () => false, + handleInstall: async () => {}, handleIsInstalled: async () => false, }, -}; +} as RTEOptions; test.serial('execute should run a job and return the result', async (t) => { const state = { plan, - }; + } as WorkflowState; - const options = { - noCompile: true, - autoinstall: { - handleInstall: async () => false, - handleIsInstalled: async () => false, - }, - }; + const context = createContext({ state, options }); - const result = await execute(api, state, options); + const result = await execute(context); t.is(result, 22); }); // we can check the state object after each of these is returned test.serial('should emit a workflow-start event', async (t) => { + const state = { + plan, + } as WorkflowState; let workflowStart; - api.once(WORKFLOW_START, (evt) => (workflowStart = evt)); + const context = createContext({ state, options }); - const state = { - plan, - }; + context.once(WORKFLOW_START, (evt) => (workflowStart = evt)); - await execute(api, state, options); + await execute(context); // No need to do a deep test of the event payload here t.is(workflowStart.workflowId, 'x'); @@ -76,14 +74,15 @@ test.serial('should emit a workflow-start event', async (t) => { test.serial('should emit a workflow-complete event', async (t) => { let workflowComplete; - - api.once(WORKFLOW_COMPLETE, (evt) => (workflowComplete = evt)); - const state = { plan, - }; + } as WorkflowState; + + const context = createContext({ state, options }); - await execute(api, state, options); + context.once(WORKFLOW_COMPLETE, (evt) => (workflowComplete = evt)); + + await execute(context); t.is(workflowComplete.workflowId, 'x'); t.is(workflowComplete.state, 22); @@ -91,9 +90,6 @@ test.serial('should emit a workflow-complete event', async (t) => { test.serial('should emit a log event', async (t) => { let workflowLog; - - api.once(WORKFLOW_LOG, (evt) => (workflowLog = evt)); - const plan = { id: 'y', jobs: [ @@ -105,9 +101,12 @@ test.serial('should emit a log event', async (t) => { const state = { id: 'y', plan, - }; + } as WorkflowState; + + const context = createContext({ state, options }); + context.once(WORKFLOW_LOG, (evt) => (workflowLog = evt)); - await execute(api, state, options); + await execute(context); t.is(workflowLog.workflowId, 'y'); t.is(workflowLog.message[0], 'hi'); diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 55f4043e3..f897b42a5 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -4,48 +4,42 @@ import { EventEmitter } from 'node:events'; import * as e from '../../src/events'; import { createMockLogger } from '@openfn/logger'; import { log, workflowComplete, workflowStart } from '../../src/api/lifecycle'; -import { EngineAPI, WorkflowState } from '../../src/types'; +import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; -// TODO this probably wants unit testing -// is it even worth mocking it? -const createMockAPI = (): EngineAPI => { +const createContext = ({ state }: Partial = {}) => { const api = new EventEmitter(); - - Object.assign(api, { + return Object.assign(api, { logger: createMockLogger(), - getWorkflowState: () => {}, - setWorkflowState: () => {}, - }); - - return api as EngineAPI; + state: state || {}, + // logger: not used + }) as unknown as ExecutionContext; }; test(`workflowStart: emits ${e.WORKFLOW_START}`, (t) => { return new Promise((done) => { const workflowId = 'a'; - const api = createMockAPI(); - const state = {} as WorkflowState; + const context = createContext(); const event = { workflowId, threadId: '123' }; - api.on(e.WORKFLOW_START, (evt) => { + context.on(e.WORKFLOW_START, (evt) => { t.deepEqual(evt, { workflowId }); done(); }); - workflowStart(api, state, event); + workflowStart(context, event); }); }); test('onWorkflowStart: updates state', (t) => { const workflowId = 'a'; - const api = createMockAPI(); - const state = {} as WorkflowState; + const context = createContext(); const event = { workflowId, threadId: '123' }; - workflowStart(api, state, event); + workflowStart(context, event); + const { state } = context; t.is(state.status, 'running'); t.is(state.duration, -1); t.is(state.threadId, '123'); @@ -60,21 +54,21 @@ test(`workflowComplete: emits ${e.WORKFLOW_COMPLETE}`, (t) => { const workflowId = 'a'; const result = { a: 777 }; - const api = createMockAPI(); const state = { startTime: Date.now() - 1000, } as WorkflowState; + const context = createContext({ state }); - const event = { workflowId, state: result }; + const event = { workflowId, state: result, threadId: '1' }; - api.on(e.WORKFLOW_COMPLETE, (evt) => { + context.on(e.WORKFLOW_COMPLETE, (evt) => { t.is(evt.workflowId, workflowId); t.deepEqual(evt.state, result); t.assert(evt.duration > 0); done(); }); - workflowComplete(api, state, event); + workflowComplete(context, event); }); }); @@ -82,13 +76,13 @@ test('workflowComplete: updates state', (t) => { const workflowId = 'a'; const result = { a: 777 }; - const api = createMockAPI(); const state = { startTime: Date.now() - 1000, } as WorkflowState; - const event = { workflowId, state: result }; + const context = createContext({ state }); + const event = { workflowId, state: result, threadId: '1' }; - workflowComplete(api, state, event); + workflowComplete(context, event); t.is(state.status, 'done'); t.assert(state.duration! > 0); @@ -99,10 +93,10 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { return new Promise((done) => { const workflowId = 'a'; - const api = createMockAPI(); const state = { id: workflowId, } as WorkflowState; + const context = createContext({ state }); const event = { workflowId, @@ -114,7 +108,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { }, }; - api.on(e.WORKFLOW_LOG, (evt) => { + context.on(e.WORKFLOW_LOG, (evt) => { t.deepEqual(evt, { workflowId: state.id, ...event.message, @@ -122,6 +116,6 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { done(); }); - log(api, state, event); + log(context, event); }); }); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 2cd5c46be..7c31c8cf7 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -102,7 +102,7 @@ test('execute with test worker and trigger workflow-complete', (t) => { }); }); -test.only('listen to workflow-complete', (t) => { +test('listen to workflow-complete', (t) => { return new Promise((done) => { const p = path.resolve('test/worker-functions.js'); const engine = createEngine( From 78cc25ec9353425d2fcacaacf66aeeeae8da70a2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 11 Oct 2023 16:49:01 +0100 Subject: [PATCH 135/232] engine: extra test --- packages/engine-multi/test/engine.test.ts | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 7c31c8cf7..2ba77e0bd 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -102,6 +102,50 @@ test('execute with test worker and trigger workflow-complete', (t) => { }); }); +test('execute does not return internal state stuff', (t) => { + return new Promise((done) => { + const p = path.resolve('test/worker-functions.js'); + const engine = createEngine( + { + logger, + noCompile: true, + autoinstall: { + handleIsInstalled: async () => true, + }, + }, + p + ); + + const plan = { + id: 'a', + jobs: [ + { + expression: '22', + }, + ], + }; + + const result = engine.execute(plan); + // Execute returns an event listener + t.truthy(result.on); + t.truthy(result.once); + t.truthy(result.off); + + // ...but not en event emitter + t.falsy(result['emit']); + t.falsy(result['dispatch']); + + // and no other execution context + t.falsy(result['state']); + t.falsy(result['logger']); + t.falsy(result['callWorker']); + t.falsy(result['options']); + + done(); + // TODO is this still running? Does it matter? + }); +}); + test('listen to workflow-complete', (t) => { return new Promise((done) => { const p = path.resolve('test/worker-functions.js'); From b18c1c0176bdebd639a8762866597a5c11ea8f03 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 11 Oct 2023 19:26:26 +0100 Subject: [PATCH 136/232] engine: tidying and typings --- packages/engine-multi/src/__api.ts | 123 ----------- packages/engine-multi/src/__engine.ts | 207 ------------------ packages/engine-multi/src/api/create-state.ts | 11 +- packages/engine-multi/src/engine.ts | 56 +++-- packages/engine-multi/src/types.d.ts | 27 ++- 5 files changed, 63 insertions(+), 361 deletions(-) delete mode 100644 packages/engine-multi/src/__api.ts delete mode 100644 packages/engine-multi/src/__engine.ts diff --git a/packages/engine-multi/src/__api.ts b/packages/engine-multi/src/__api.ts deleted file mode 100644 index 9b5f58112..000000000 --- a/packages/engine-multi/src/__api.ts +++ /dev/null @@ -1,123 +0,0 @@ -import createLogger, { JSONLog, Logger } from '@openfn/logger'; -import { EngineAPI, EngineEvents, EventHandler } from './types'; -import { EventEmitter } from 'node:events'; -import { WORKFLOW_COMPLETE, WORKFLOW_START } from './events'; - -import initWorkers from './api/call-worker'; -import { RTEOptions } from './__engine'; - -// For each workflow, create an API object with its own event emitter -// this is a bt wierd - what if the emitter went on state instead? -const createWorkflowEvents = (api: EngineAPI) => { - //create a bespoke event emitter - const events = new EventEmitter(); - - // TODO need this in closure really - // listeners[workflowId] = events; - - // proxy all events to the main emitter - // uh actually there may be no point in this - function proxy(event: string) { - events.on(event, (evt) => { - // ensure the attempt id is on the event - evt.workflowId = workflowId; - const newEvt = { - ...evt, - workflowId: workflowId, - }; - - api.emit(event, newEvt); - }); - } - proxy(WORKFLOW_START); - proxy(WORKFLOW_COMPLETE); - proxy(JOB_START); - proxy(JOB_COMPLETE); - proxy(LOG); - - return events; -}; - -// This creates the internal API -// tbh this is actually the engine, right, this is where stuff happens -// the api file is more about the public api I think -const createAPI = (options: RTEOptions) => { - const listeners = {}; - - // but this is more like an internal api, right? - // maybe this is like the workflow context - const state = { - workflows: {}, - }; // TODO maybe this sits in another file - // the state has apis for getting/setting workflow state - // maybe actually each execute setsup its own state object - - // TODO I think there's an internal and external API - // api is the external thing that other people call - // engine is the internal thing - const api = new EventEmitter() as EngineAPI; - - api.logger = options.logger || createLogger('RTE', { level: 'debug' }); - - api.registerWorkflow = (state) => { - state.workflows[plan.id] = state; - }; - - // what if this returns a bespoke event listener? - // i don't need to to execute(); listen, i can just excute - // it's kinda slick but you still need two lines of code and it doesn't buy anyone anything - // also this is nevver gonna get used externally so it doesn't need to be slick - api.execute = (executionPlan) => { - const workflowId = plan.id; - - // Pull options out of the plan so that all options are in one place - const { options, ...plan } = executionPlan; - - // initial state for this workflow run - // TODO let's create a util function for this (nice for testing) - const state = createState(plan); - // the engine does need to be able to report on the state of each workflow - api.registerWorkflow(state); - - const events = createWorkflowEvents(api); - - listeners[workflowId] = events; - - // this context API thing is the internal api / engine - // each has a bespoke event emitter but otherwise a common interface - const contextAPI: EngineAPI = { - ...api, - ...events, - }; - - execute(contextAPI, state, options); - - // return the event emitter (not the full engine API though) - return events; - }; - - // // how will this actually work? - // api.listen = ( - // attemptId: string, - // listeners: Record - // ) => { - // // const handler = (eventName) => { - // // if () - // // } - // const events = listeners[workflowId]; - // for (const evt of listeners) { - // events.on(evt, listeners[evt]); - // } - - // // TODO return unsubscribe handle - // }; - - // we can have global reporting like this - api.getStatus = (workflowId) => state.workflows[workflowId].status; - - initWorkers(api); - - return api; -}; - -export default createAPI; diff --git a/packages/engine-multi/src/__engine.ts b/packages/engine-multi/src/__engine.ts deleted file mode 100644 index 6d3fad724..000000000 --- a/packages/engine-multi/src/__engine.ts +++ /dev/null @@ -1,207 +0,0 @@ -import path from 'node:path'; -import crypto from 'node:crypto'; -import { fileURLToPath } from 'url'; -import { EventEmitter } from 'node:events'; -import workerpool from 'workerpool'; -import { ExecutionPlan } from '@openfn/runtime'; -import createLogger, { JSONLog, Logger } from '@openfn/logger'; - -import * as e from './events'; -import compile from './api/compile'; -import execute from './runners/execute'; -import autoinstall, { AutoinstallOptions } from './api/autoinstall'; - -export type State = any; // TODO I want a nice state def with generics - -// Archive of every workflow we've run -// Fine to just keep in memory for now -type WorkflowStats = { - id: string; - name?: string; // TODO what is name? this is irrelevant? - status: 'pending' | 'done' | 'err'; - startTime?: number; - threadId?: number; - duration?: number; - error?: string; - result?: any; // State - plan: ExecutionPlan; -}; - -type Resolver = (id: string) => Promise; - -// A list of helper functions which basically resolve ids into JSON -// to lazy load assets -export type LazyResolvers = { - credentials?: Resolver; - state?: Resolver; - expressions?: Resolver; -}; - -export type RTEOptions = { - resolvers?: LazyResolvers; - logger?: Logger; - workerPath?: string; - repoDir?: string; - noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? - - autoinstall: AutoinstallOptions; -}; - -const createRTM = function (serverId?: string, options: RTEOptions = {}) { - const { noCompile } = options; - let { repoDir, workerPath } = options; - - const id = serverId || crypto.randomUUID(); - const logger = options.logger || createLogger('RTM', { level: 'debug' }); - - const allWorkflows: Map = new Map(); - const activeWorkflows: string[] = []; - - // TODO we want to get right down to this - - // Create the internal API - const api = createApi(options); - - // Return the external API - return { - execute: api.execute, - listen: api.listen, - }; - - // let resolvedWorkerPath; - // if (workerPath) { - // // If a path to the worker has been passed in, just use it verbatim - // // We use this to pass a mock worker for testing purposes - // resolvedWorkerPath = workerPath; - // } else { - // // By default, we load ./worker.js but can't rely on the working dir to find it - // const dirname = path.dirname(fileURLToPath(import.meta.url)); - // resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); - // } - // const workers = workerpool.pool(resolvedWorkerPath); - - const events = new EventEmitter(); - - if (!repoDir) { - if (process.env.OPENFN_RTE_REPO_DIR) { - repoDir = process.env.OPENFN_RTE_REPO_DIR; - } else { - repoDir = '/tmp/openfn/repo'; - logger.warn('Using default repodir'); - logger.warn( - 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' - ); - } - } - logger.info('repoDir set to ', repoDir); - - // const onWorkflowStarted = (workflowId: string, threadId: number) => { - // logger.info('starting workflow ', workflowId); - // const workflow = allWorkflows.get(workflowId)!; - - // if (workflow.startTime) { - // // TODO this shouldn't throw.. but what do we do? - // // We shouldn't run a workflow that's been run - // // Every workflow should have a unique id - // // maybe the RTM doesn't care about this - // throw new Error(`Workflow with id ${workflowId} is already started`); - // } - // workflow.startTime = new Date().getTime(); - // workflow.duration = -1; - // workflow.threadId = threadId; - // activeWorkflows.push(workflowId); - - // // forward the event on to any external listeners - // events.emit(e.WORKFLOW_START, { - // workflowId, - // // Should we publish anything else here? - // }); - // }; - - // const completeWorkflow = (workflowId: string, state: any) => { - // logger.success('complete workflow ', workflowId); - // logger.info(state); - // if (!allWorkflows.has(workflowId)) { - // throw new Error(`Workflow with id ${workflowId} is not defined`); - // } - // const workflow = allWorkflows.get(workflowId)!; - // workflow.status = 'done'; - // workflow.result = state; - // workflow.duration = new Date().getTime() - workflow.startTime!; - // const idx = activeWorkflows.findIndex((id) => id === workflowId); - // activeWorkflows.splice(idx, 1); - - // // forward the event on to any external listeners - // events.emit(e.WORKFLOW_COMPLETE, { - // id: workflowId, - // duration: workflow.duration, - // state, - // }); - // }; - - // // Catch a log coming out of a job within a workflow - // // Includes runtime logging (is this right?) - // const onWorkflowLog = (workflowId: string, message: JSONLog) => { - // // Seamlessly proxy the log to the local stdout - // // TODO runtime logging probably needs to be at info level? - // // Debug information is mostly irrelevant for lightning - // const newMessage = { - // ...message, - // // Prefix the job id in all local jobs - // // I'm sure there are nicer, more elegant ways of doing this - // message: [`[${workflowId}]`, ...message.message], - // }; - // logger.proxy(newMessage); - // events.emit(e.WORKFLOW_LOG, { - // workflowId, - // message, - // }); - // }; - - // How much of this happens inside the worker? - // Shoud the main thread handle compilation? Has to if we want to cache - // Unless we create a dedicated compiler worker - // TODO error handling, timeout - const handleExecute = async (plan: ExecutionPlan) => { - const options = { - repoDir, - }; - const context = { plan, logger, workers, options /* api */ }; - - logger.debug('Executing workflow ', plan.id); - - allWorkflows.set(plan.id!, { - id: plan.id!, - status: 'pending', - plan, - }); - - const adaptorPaths = await autoinstall(context); - - if (!noCompile) { - context.plan = await compile(context); - } - - logger.debug('workflow compiled ', plan.id); - const result = await execute(context, adaptorPaths, { - start: onWorkflowStarted, - log: onWorkflowLog, - }); - completeWorkflow(plan.id!, result); - - logger.debug('finished executing workflow ', plan.id); - // Return the result - // Note that the mock doesn't behave like ths - // And tbf I don't think we should keep the promise open - there's no point? - return result; - }; - - return { - id, - on: (type: string, fn: (...args: any[]) => void) => events.on(type, fn), - once: (type: string, fn: (...args: any[]) => void) => events.once(type, fn), - execute: handleExecute, - }; -}; - -export default createRTM; diff --git a/packages/engine-multi/src/api/create-state.ts b/packages/engine-multi/src/api/create-state.ts index d25b07776..7e1c538aa 100644 --- a/packages/engine-multi/src/api/create-state.ts +++ b/packages/engine-multi/src/api/create-state.ts @@ -1,5 +1,8 @@ -export default (plan) => ({ - id: plan.id, +import { ExecutionPlan } from '@openfn/runtime'; +import { WorkflowState } from '../types'; + +export default (plan: ExecutionPlan, options = {}): WorkflowState => ({ + id: plan.id!, status: 'pending', plan, @@ -9,7 +12,9 @@ export default (plan) => ({ error: undefined, result: undefined, - // yeah not sure about options right now + // this is wf-specific options + // but they should be on context, rather than state + options, // options: { // ...options, // repoDir, diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 2baff1b40..fc8c0a611 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -14,7 +14,10 @@ import type { EngineEvents, EventHandler, WorkflowState, + CallWorker, + ExecutionContextConstructor, } from './types'; +import { Logger } from '@openfn/logger'; // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? @@ -54,11 +57,21 @@ class Engine extends EventEmitter {} // TODO this is actually the api that each execution gets // its nice to separate that from the engine a bit class ExecutionContext extends EventEmitter { - constructor(workflowState, logger, callWorker, options) { + state: WorkflowState; + logger: Logger; + callWorker: CallWorker; + options: RTEOptions; + + constructor({ + state, + logger, + callWorker, + options, + }: ExecutionContextConstructor) { super(); this.logger = logger; this.callWorker = callWorker; - this.state = workflowState; + this.state = state; this.options = options; } } @@ -66,12 +79,10 @@ class ExecutionContext extends EventEmitter { // This creates the internal API // tbh this is actually the engine, right, this is where stuff happens // the api file is more about the public api I think +// TOOD options MUST have a logger const createEngine = (options: RTEOptions, workerPath?: string) => { - // TODO this is probably states now, although contexts sort of subsumes it - const allWorkflows: Map = new Map(); - - // TODO this contexts now - const listeners = {}; + const states: Record = {}; + const contexts: Record = {}; // TODO I think this is for later //const activeWorkflows: string[] = []; @@ -85,7 +96,7 @@ const createEngine = (options: RTEOptions, workerPath?: string) => { const dirname = path.dirname(fileURLToPath(import.meta.url)); resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); } - options.logger.debug('Loading workers from ', resolvedWorkerPath); + options.logger!.debug('Loading workers from ', resolvedWorkerPath); const engine = new Engine() as EngineAPI; @@ -99,35 +110,40 @@ const createEngine = (options: RTEOptions, workerPath?: string) => { const registerWorkflow = (plan: ExecutionPlan) => { // TODO throw if already registered? const state = createState(plan); - allWorkflows[state.id] = state; + states[state.id] = state; return state; }; - const getWorkflowState = (workflowId: string) => allWorkflows[workflowId]; + const getWorkflowState = (workflowId: string) => states[workflowId]; - const getWorkflowStatus = (workflowId: string) => - allWorkflows[workflowId]?.status; + const getWorkflowStatus = (workflowId: string) => states[workflowId]?.status; // TODO are we totally sure this takes a standard xplan? + // Well, it MUST have an ID or there's trouble const executeWrapper = (plan: ExecutionPlan) => { + const workflowId = plan.id!; + // TODO throw if plan is invalid + // Wait, don't throw because the server will die + // Maybe return null instead const state = registerWorkflow(plan); - const context = new ExecutionContext( + const context = new ExecutionContext({ state, - options.logger, - engine.callWorker, - options - ); + logger: options.logger!, + callWorker: engine.callWorker, + options, + }); - listeners[plan.id] = createWorkflowEvents(engine, context, engine, plan.id); + contexts[workflowId] = createWorkflowEvents(engine, context, workflowId); + // TODO typing between the class and interface isn't right execute(context); // hmm. Am I happy to pass the internal workflow state OUT of the handler? // I'd rather have like a proxy emitter or something // also I really only want passive event handlers, I don't want interference from outside return { - on: (...args) => context.on(...args), + on: (...args: any[]) => context.on(...args), once: context.once, off: context.off, }; @@ -138,7 +154,7 @@ const createEngine = (options: RTEOptions, workerPath?: string) => { workflowId: string, handlers: Record ) => { - const events = listeners[workflowId]; + const events = contexts[workflowId]; for (const evt in handlers) { events.on(evt, handlers[evt]); } diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index d5104d9ec..793b50136 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -116,22 +116,33 @@ export type WorkflowState = { error?: string; result?: any; // State plan: ExecutionPlan; // this doesn't include options - options: any; // TODO this is general engine options and workflow options + options: any; // TODO this is wf specific options, like logging policy +}; + +export type CallWorker = ( + task: string, + args: any[] = [], + events: any = {} +) => workerpool.Promise; + +export type ExecutionContextConstructor = { + state: WorkflowState; + logger: Logger; + callWorker: CallWorker; + options: RTEOptions; }; -// this is the internal engine API export interface ExecutionContext extends EventEmitter { + constructor(args: ExecutionContextConstructor); options: RTEOptions; // TODO maybe. bring them in here? state: WorkflowState; logger: Logger; - callWorker: ( - task: string, - args: any[] = [], - events: any = {} - ) => workerpool.Promise; + callWorker: CallWorker; } -export interface EngineAPI extends EventEmitter {} +export interface EngineAPI extends EventEmitter { + callWorker: CallWorker; +} interface RuntimeEngine extends EventEmitter { //id: string // human readable instance id From 7cda1fdf116b13235f5120c181a170d94c096dc8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 12 Oct 2023 15:12:09 +0100 Subject: [PATCH 137/232] engine: add public api wrapper --- packages/engine-multi/package.json | 4 +- packages/engine-multi/src/api.ts | 32 +++++--- packages/engine-multi/src/engine.ts | 19 ++++- packages/engine-multi/src/events.ts | 5 +- packages/engine-multi/src/types.d.ts | 5 +- packages/engine-multi/test/api.test.ts | 96 ++++++++++++++++++++--- packages/engine-multi/test/engine.test.ts | 23 ------ 7 files changed, 128 insertions(+), 56 deletions(-) diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 7f0f30877..9f17bf2b9 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -8,8 +8,8 @@ "scripts": { "test": "pnpm ava", "_test:types": "pnpm tsc --noEmit --project tsconfig.json", - "_build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting", - "build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", + "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting --transpile-only", + "_build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch" }, "author": "Open Function Group ", diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 78f3f5130..2941f96ad 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -5,6 +5,7 @@ import crypto from 'node:crypto'; import { ExecutionPlan } from '@openfn/runtime'; import createLogger, { JSONLog, Logger } from '@openfn/logger'; +import createEngine from './engine'; import type { AutoinstallOptions } from './api/autoinstall'; export type State = any; // TODO I want a nice state def with generics @@ -36,16 +37,19 @@ export type LazyResolvers = { export type RTEOptions = { resolvers?: LazyResolvers; logger?: Logger; - workerPath?: string; // TODO maybe the public API doesn't expose this repoDir?: string; + noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? + compile?: { + skip: true; + }; - autoinstall: AutoinstallOptions; + autoinstall?: AutoinstallOptions; }; // Create the engine and handle user-facing stuff, like options parsing // and defaulting -const createAPI = function (serverId?: string, options: RTEOptions = {}) { +const createAPI = function (options: RTEOptions = {}) { let { repoDir } = options; const logger = options.logger || createLogger('RTE', { level: 'debug' }); @@ -64,20 +68,30 @@ const createAPI = function (serverId?: string, options: RTEOptions = {}) { logger.info('repoDir set to ', repoDir); - // TODO I dunno, does the engine have an id? - // I think that's a worker concern, especially - // as there's a 1:1 worker:engine mapping - // const id = serverId || crypto.randomUUID(); + const engineOptions = { + logger, + resolvers: options.resolvers, // TODO should probably default these? + repoDir, - // TODO we want to get right down to this + // TODO should map this down into compile. + noCompile: options.compile?.skip ?? false, + // TODO should we disable autoinstall overrides? + autoinstall: options.autoinstall, + }; // Create the internal API - const engine = createApi(options); + // TMP: use the mock worker for now + const engine = createEngine( + engineOptions, + path.resolve('dist/mock-worker.js') + ); // Return the external API return { execute: engine.execute, listen: engine.listen, + + // TODO what about a general on or once? }; }; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index fc8c0a611..0e6132b0c 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -8,7 +8,7 @@ import initWorkers from './api/call-worker'; import createState from './api/create-state'; import execute from './api/execute'; -import type { RTEOptions } from './api'; +import type { LazyResolvers, RTEOptions } from './api'; import type { EngineAPI, EngineEvents, @@ -18,6 +18,7 @@ import type { ExecutionContextConstructor, } from './types'; import { Logger } from '@openfn/logger'; +import { AutoinstallOptions } from './api/autoinstall'; // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? @@ -76,11 +77,22 @@ class ExecutionContext extends EventEmitter { } } +// The enigne is way more strict about options +export type EngineOptions = { + repoDir: string; + logger: Logger; + + resolvers: LazyResolvers; + + compile?: {}; // TODO + autoinstall?: AutoinstallOptions; +}; + // This creates the internal API // tbh this is actually the engine, right, this is where stuff happens // the api file is more about the public api I think // TOOD options MUST have a logger -const createEngine = (options: RTEOptions, workerPath?: string) => { +const createEngine = (options: EngineOptions, workerPath?: string) => { const states: Record = {}; const contexts: Record = {}; // TODO I think this is for later @@ -152,7 +164,8 @@ const createEngine = (options: RTEOptions, workerPath?: string) => { const listen = ( workflowId: string, - handlers: Record + //handlers: Partial> + handlers: Record ) => { const events = contexts[workflowId]; for (const evt in handlers) { diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 1d04bf2b6..0304d6c4d 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -3,8 +3,9 @@ // TODO mayberename event constants import { JSONLog } from '@openfn/logger'; -// Top level API events - these are what the engine publishes - +// Top level API events - these are what the engine publishes externally +// should it just be start, log, job-start, job-complete, end etc? +// What about engine-level logging? CLI-level stuff? export const WORKFLOW_START = 'workflow-start'; export const WORKFLOW_COMPLETE = 'workflow-complete'; diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index 793b50136..1dbee74e4 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -149,10 +149,7 @@ interface RuntimeEngine extends EventEmitter { // actually I think the id is on the worker, not the engine // TODO should return an unsubscribe hook - listen( - attemptId: string, - listeners: Record - ): void; + listen(attemptId: string, listeners: any): void; // TODO return a promise? // Kinda convenient but not actually needed diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index e0430208d..809de9de4 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -1,21 +1,91 @@ import test from 'ava'; -import createAPI from '../src/__api'; +import createAPI from '../src/api'; +import { createMockLogger } from '@openfn/logger'; // thes are tests on the public api functions generally -// so these are very high level tests +// so these are very high level tests and don't allow mock workers or anything -// no need to test the event stuff - startworkflow etc -// maybe we can check the keys exist, although we'll quickly know if we dont +const logger = createMockLogger(); -test.todo('execute'); -test.todo('execute should return an event emitter'); -test.todo('execute should proxy events'); -test.todo('listen'); -test.todo('log'); +test.afterEach(() => { + logger._reset(); +}); -// test('callWorker', (t) => { -// const api = createAPI(); +test('create a default engine api without throwing', (t) => { + createAPI(); + t.pass(); +}); -// t.pass(); -// }); +test('create an engine api with options without throwing', (t) => { + createAPI({ logger }); + + // just a token test to see if the logger is accepted and used + t.assert(logger._history.length > 0); +}); + +test('create an engine api with a limited surface', (t) => { + const api = createAPI({ logger }); + const keys = Object.keys(api); + + // TODO the api will actually probably get a bit bigger than this + t.deepEqual(keys, ['execute', 'listen']); +}); + +test('execute should return an event listener and receive workflow-complete', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + // Disable compilation + compile: { + skip: true, + }, + }); + + const plan = { + id: 'a', + jobs: [ + { + expression: 's => s', + // with no adaptor it shouldn't try to autoinstall + }, + ], + }; + + const listener = api.execute(plan); + listener.on('workflow-complete', () => { + t.pass('workflow completed'); + done(); + }); + }); +}); + +test('should listen to workflow-complete', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + // Disable compilation + compile: { + skip: true, + }, + }); + + const plan = { + id: 'a', + jobs: [ + { + expression: 's => s', + // with no adaptor it shouldn't try to autoinstall + }, + ], + }; + + api.execute(plan); + api.listen(plan.id, { + 'workflow-complete': () => { + t.pass('workflow completed'); + done(); + }, + }); + }); +}); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 2ba77e0bd..36ad1f366 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -179,26 +179,3 @@ test('listen to workflow-complete', (t) => { }); }); }); - -test.skip('Should run a mock job with a simple return value', async (t) => { - const state = { data: { x: 1 } }; - const rtm = Manager('x', options); - const plan = createPlan({ - expression: `() => (${JSON.stringify(state)})`, - }); - const result = await rtm.execute(plan); - t.deepEqual(result, state); -}); - -test.skip('Should not explode if no adaptor is passed', async (t) => { - const state = { data: { x: 1 } }; - const rtm = Manager('x', options); - const plan = createPlan({ - expression: `() => (${JSON.stringify(state)})`, - }); - - // @ts-ignore - delete plan.jobs[0].adaptor; - const result = await rtm.execute(plan); - t.deepEqual(result, state); -}); From 8e4a685a653bcb56a9c51b5d90cb4f56521a973a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 12 Oct 2023 15:28:55 +0100 Subject: [PATCH 138/232] engine: fix types and restore types test --- packages/engine-multi/package.json | 2 +- packages/engine-multi/src/api.ts | 23 ++++------------ packages/engine-multi/src/api/autoinstall.ts | 10 +++++-- packages/engine-multi/src/api/compile.ts | 29 +++++++++++--------- packages/engine-multi/src/api/execute.ts | 17 +++++++----- packages/engine-multi/src/api/lifecycle.ts | 2 -- packages/engine-multi/src/engine.ts | 17 ++++++------ packages/engine-multi/src/events.ts | 4 +-- packages/engine-multi/src/types.d.ts | 2 +- packages/engine-multi/src/worker-helper.ts | 2 +- 10 files changed, 54 insertions(+), 54 deletions(-) diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 9f17bf2b9..065afcf1b 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -7,7 +7,7 @@ "private": true, "scripts": { "test": "pnpm ava", - "_test:types": "pnpm tsc --noEmit --project tsconfig.json", + "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting --transpile-only", "_build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch" diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 2941f96ad..8784704f4 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -1,29 +1,14 @@ // Creates the public/external API to the runtime +// Basically a thin wrapper, with validation, around the engine import path from 'node:path'; -import crypto from 'node:crypto'; -import { ExecutionPlan } from '@openfn/runtime'; -import createLogger, { JSONLog, Logger } from '@openfn/logger'; +import createLogger, { Logger } from '@openfn/logger'; import createEngine from './engine'; import type { AutoinstallOptions } from './api/autoinstall'; export type State = any; // TODO I want a nice state def with generics -// Archive of every workflow we've run -// Fine to just keep in memory for now -type WorkflowStats = { - id: string; - name?: string; // TODO what is name? this is irrelevant? - status: 'pending' | 'done' | 'err'; - startTime?: number; - threadId?: number; - duration?: number; - error?: string; - result?: any; // State - plan: ExecutionPlan; -}; - type Resolver = (id: string) => Promise; // A list of helper functions which basically resolve ids into JSON @@ -66,6 +51,10 @@ const createAPI = function (options: RTEOptions = {}) { } } + // re logging, for example, where does this go? + // it's not an attempt log + // it probably shouldnt be sent to the worker + // but it is an important bit of debugging logger.info('repoDir set to ', repoDir); const engineOptions = { diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 026145093..1aa8eab0d 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -8,8 +8,9 @@ import { loadRepoPkg, } from '@openfn/runtime'; import { install as runtimeInstall } from '@openfn/runtime'; + import type { Logger } from '@openfn/logger'; -import { EngineAPI, ExecutionContext, WorkflowState } from '../types'; +import type { ExecutionContext } from '../types'; // none of these options should be on the plan actually export type AutoinstallOptions = { @@ -36,7 +37,12 @@ const autoinstall = async (context: ExecutionContext): Promise => { let didValidateRepo = false; const { skipRepoValidation } = autoinstallOptions; - if (!skipRepoValidation && !didValidateRepo && repoDir) { + if (!repoDir) { + logger.warn('WARNING: skipping autoinstall because repoDir is not set'); + return {}; + } + + if (!skipRepoValidation && !didValidateRepo) { // TODO what if this throws? // Whole server probably needs to crash, so throwing is probably appropriate await ensureRepo(repoDir, logger); diff --git a/packages/engine-multi/src/api/compile.ts b/packages/engine-multi/src/api/compile.ts index 469e8af1c..532060f2b 100644 --- a/packages/engine-multi/src/api/compile.ts +++ b/packages/engine-multi/src/api/compile.ts @@ -3,7 +3,7 @@ // being compiled twice import type { Logger } from '@openfn/logger'; -import compile, { preloadAdaptorExports } from '@openfn/compiler'; +import compile, { preloadAdaptorExports, Options } from '@openfn/compiler'; import { getModulePath } from '@openfn/runtime'; import { ExecutionContext } from '../types'; @@ -19,9 +19,9 @@ export default async (context: ExecutionContext) => { if (job.expression) { job.expression = await compileJob( job.expression as string, - job.adaptor, // TODO need to expand this. Or do I? + logger, repoDir, - logger + job.adaptor // TODO need to expand this. Or do I? ); } } @@ -41,22 +41,25 @@ const stripVersionSpecifier = (specifier: string) => { const compileJob = async ( job: string, - adaptor: string, - repoDir: string, - logger: Logger + logger: Logger, + repoDir?: string, + adaptor?: string ) => { - // TODO I probably dont want to log this stuff - const pathToAdaptor = await getModulePath(adaptor, repoDir, logger); - const exports = await preloadAdaptorExports(pathToAdaptor!, false, logger); - const compilerOptions = { + const compilerOptions: Options = { logger, - ['add-imports']: { + }; + + if (adaptor && repoDir) { + // TODO I probably dont want to log this stuff + const pathToAdaptor = await getModulePath(adaptor, repoDir, logger); + const exports = await preloadAdaptorExports(pathToAdaptor!, false, logger); + compilerOptions['add-imports'] = { adaptor: { name: stripVersionSpecifier(adaptor), exports, exportAll: true, }, - }, - }; + }; + } return compile(job, compilerOptions); }; diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 553bdb229..d77204c11 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -17,21 +17,24 @@ const execute = async (context: ExecutionContext) => { await compile(context); const events = { - [e.WORKFLOW_START]: (evt) => { + // TODO typings + [e.WORKFLOW_START]: (evt: any) => { workflowStart(context, evt); }, - [e.WORKFLOW_COMPLETE]: (evt) => { + [e.WORKFLOW_COMPLETE]: (evt: any) => { workflowComplete(context, evt); }, - [e.WORKFLOW_LOG]: (evt) => { + [e.WORKFLOW_LOG]: (evt: any) => { log(context, evt); }, }; - return callWorker('run', [state.plan, adaptorPaths], events).catch((e) => { - // TODO what about errors then? - logger.error(e); - }); + return callWorker('run', [state.plan, adaptorPaths], events).catch( + (e: any) => { + // TODO what about errors then? + logger.error(e); + } + ); }; export default execute; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 6219e527b..2f789ae88 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -1,11 +1,9 @@ import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from '../events'; import { - EngineAPI, ExecutionContext, WorkerCompletePayload, WorkerLogPayload, WorkerStartPayload, - WorkflowState, } from '../types'; export const workflowStart = ( diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 0e6132b0c..6f0dcb4bd 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -8,10 +8,9 @@ import initWorkers from './api/call-worker'; import createState from './api/create-state'; import execute from './api/execute'; -import type { LazyResolvers, RTEOptions } from './api'; +import type { LazyResolvers } from './api'; import type { EngineAPI, - EngineEvents, EventHandler, WorkflowState, CallWorker, @@ -61,7 +60,7 @@ class ExecutionContext extends EventEmitter { state: WorkflowState; logger: Logger; callWorker: CallWorker; - options: RTEOptions; + options: EngineOptions; constructor({ state, @@ -77,12 +76,12 @@ class ExecutionContext extends EventEmitter { } } -// The enigne is way more strict about options +// The engine is way more strict about options export type EngineOptions = { repoDir: string; logger: Logger; - resolvers: LazyResolvers; + resolvers?: LazyResolvers; compile?: {}; // TODO autoinstall?: AutoinstallOptions; @@ -149,15 +148,17 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { contexts[workflowId] = createWorkflowEvents(engine, context, workflowId); // TODO typing between the class and interface isn't right + // @ts-ignore execute(context); // hmm. Am I happy to pass the internal workflow state OUT of the handler? // I'd rather have like a proxy emitter or something // also I really only want passive event handlers, I don't want interference from outside return { - on: (...args: any[]) => context.on(...args), - once: context.once, - off: context.off, + on: (evt: string, fn: (...args: any[]) => void) => context.on(evt, fn), + once: (evt: string, fn: (...args: any[]) => void) => + context.once(evt, fn), + off: (evt: string, fn: (...args: any[]) => void) => context.off(evt, fn), }; // return context; }; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 0304d6c4d..86aa31f70 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -46,5 +46,5 @@ export type LogWorkflowEvent = { export type WorkflowEvent = | AcceptWorkflowEvent | CompleteWorkflowEvent - | ErrWorkflowEvent; -// | LogWorkflowEvent; + | ErrWorkflowEvent + | LogWorkflowEvent; diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index 1dbee74e4..cf0c25cb7 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -129,7 +129,7 @@ export type ExecutionContextConstructor = { state: WorkflowState; logger: Logger; callWorker: CallWorker; - options: RTEOptions; + options: EngineOptions; }; export interface ExecutionContext extends EventEmitter { diff --git a/packages/engine-multi/src/worker-helper.ts b/packages/engine-multi/src/worker-helper.ts index 0ad9cbc5a..5e94613d3 100644 --- a/packages/engine-multi/src/worker-helper.ts +++ b/packages/engine-multi/src/worker-helper.ts @@ -3,7 +3,7 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; -import createLogger, { JSONLog } from '@openfn/logger'; +import createLogger from '@openfn/logger'; import * as e from './events'; From 90eca708d083a3c3755d2a7867c2fd66e5c30937 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 12 Oct 2023 15:39:43 +0100 Subject: [PATCH 139/232] engine: tidyups --- packages/engine-multi/src/index.ts | 4 ++-- packages/engine-multi/src/{jobs => test}/slow-random.js | 0 packages/engine-multi/{ => src}/test/util.ts | 0 packages/engine-multi/{ => src}/test/worker-functions.js | 0 packages/engine-multi/src/{ => worker}/mock-worker.ts | 0 packages/engine-multi/src/{ => worker}/worker-helper.ts | 2 +- packages/engine-multi/src/{ => worker}/worker.ts | 0 packages/engine-multi/test/api/autoinstall.test.ts | 3 ++- packages/engine-multi/test/api/call-worker.test.ts | 2 +- packages/engine-multi/test/engine.test.ts | 8 ++++---- .../engine-multi/test/{jobs => test}/slow-random.test.ts | 3 ++- .../engine-multi/test/{ => worker}/mock-worker.test.ts | 4 ++-- .../engine-multi/test/{ => worker}/worker-pool.test.ts | 0 13 files changed, 14 insertions(+), 12 deletions(-) rename packages/engine-multi/src/{jobs => test}/slow-random.js (100%) rename packages/engine-multi/{ => src}/test/util.ts (100%) rename packages/engine-multi/{ => src}/test/worker-functions.js (100%) rename packages/engine-multi/src/{ => worker}/mock-worker.ts (100%) rename packages/engine-multi/src/{ => worker}/worker-helper.ts (98%) rename packages/engine-multi/src/{ => worker}/worker.ts (100%) rename packages/engine-multi/test/{jobs => test}/slow-random.test.ts (96%) rename packages/engine-multi/test/{ => worker}/mock-worker.test.ts (97%) rename packages/engine-multi/test/{ => worker}/worker-pool.test.ts (100%) diff --git a/packages/engine-multi/src/index.ts b/packages/engine-multi/src/index.ts index a85794d60..3bfff051f 100644 --- a/packages/engine-multi/src/index.ts +++ b/packages/engine-multi/src/index.ts @@ -1,3 +1,3 @@ -import createAPI from './api'; +import createEngine from './api'; -export default createAPI; +export default createEngine; diff --git a/packages/engine-multi/src/jobs/slow-random.js b/packages/engine-multi/src/test/slow-random.js similarity index 100% rename from packages/engine-multi/src/jobs/slow-random.js rename to packages/engine-multi/src/test/slow-random.js diff --git a/packages/engine-multi/test/util.ts b/packages/engine-multi/src/test/util.ts similarity index 100% rename from packages/engine-multi/test/util.ts rename to packages/engine-multi/src/test/util.ts diff --git a/packages/engine-multi/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js similarity index 100% rename from packages/engine-multi/test/worker-functions.js rename to packages/engine-multi/src/test/worker-functions.js diff --git a/packages/engine-multi/src/mock-worker.ts b/packages/engine-multi/src/worker/mock-worker.ts similarity index 100% rename from packages/engine-multi/src/mock-worker.ts rename to packages/engine-multi/src/worker/mock-worker.ts diff --git a/packages/engine-multi/src/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts similarity index 98% rename from packages/engine-multi/src/worker-helper.ts rename to packages/engine-multi/src/worker/worker-helper.ts index 5e94613d3..c3fc49aee 100644 --- a/packages/engine-multi/src/worker-helper.ts +++ b/packages/engine-multi/src/worker/worker-helper.ts @@ -5,7 +5,7 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; import createLogger from '@openfn/logger'; -import * as e from './events'; +import * as e from '../events'; function publish(event: e.WorkflowEvent) { workerpool.workerEmit(event); diff --git a/packages/engine-multi/src/worker.ts b/packages/engine-multi/src/worker/worker.ts similarity index 100% rename from packages/engine-multi/src/worker.ts rename to packages/engine-multi/src/worker/worker.ts diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index e48583977..f51f0c64f 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -28,6 +28,7 @@ const createContext = (autoinstallOpts?, jobs?: any[]) => }, }, options: { + repoDir: '.', autoinstall: autoinstallOpts || { handleInstall: mockHandleInstall, handleIsInstalled: mockIsInstalled, @@ -115,7 +116,7 @@ test.serial( async (t) => { let callCount = 0; - const mockInstall = (specififer: string) => + const mockInstall = () => new Promise((resolve) => { callCount++; setTimeout(() => resolve(), 20); diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index a4c143d66..bba65aa66 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -7,7 +7,7 @@ import { EngineAPI } from '../../src/types'; let api = {} as EngineAPI; test.before(() => { - const workerPath = path.resolve('test/worker-functions.js'); + const workerPath = path.resolve('src/test/worker-functions.js'); initWorkers(api, workerPath); }); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 36ad1f366..9cd7e1d3e 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; import path from 'node:path'; import { createMockLogger } from '@openfn/logger'; -import { createPlan } from './util'; +import { createPlan } from '../src/test/util'; import createEngine from '../src/engine'; import * as e from '../src/events'; @@ -73,7 +73,7 @@ test('use a custom worker path', (t) => { test('execute with test worker and trigger workflow-complete', (t) => { return new Promise((done) => { - const p = path.resolve('test/worker-functions.js'); + const p = path.resolve('src/test/worker-functions.js'); const engine = createEngine( { logger, @@ -104,7 +104,7 @@ test('execute with test worker and trigger workflow-complete', (t) => { test('execute does not return internal state stuff', (t) => { return new Promise((done) => { - const p = path.resolve('test/worker-functions.js'); + const p = path.resolve('src/test/worker-functions.js'); const engine = createEngine( { logger, @@ -148,7 +148,7 @@ test('execute does not return internal state stuff', (t) => { test('listen to workflow-complete', (t) => { return new Promise((done) => { - const p = path.resolve('test/worker-functions.js'); + const p = path.resolve('src/test/worker-functions.js'); const engine = createEngine( { logger, diff --git a/packages/engine-multi/test/jobs/slow-random.test.ts b/packages/engine-multi/test/test/slow-random.test.ts similarity index 96% rename from packages/engine-multi/test/jobs/slow-random.test.ts rename to packages/engine-multi/test/test/slow-random.test.ts index 67ad7827a..40a4ccc8c 100644 --- a/packages/engine-multi/test/jobs/slow-random.test.ts +++ b/packages/engine-multi/test/test/slow-random.test.ts @@ -13,7 +13,8 @@ const wait = async (time: number) => setTimeout(resolve, time); }); -const compiledJob = compile('src/jobs/slow-random.js'); +const compiledJob = compile('src/test/slow-random.js'); + test('slowmo should return a value', async (t) => { const result = (await execute(compiledJob)) as SlowMoState; diff --git a/packages/engine-multi/test/mock-worker.test.ts b/packages/engine-multi/test/worker/mock-worker.test.ts similarity index 97% rename from packages/engine-multi/test/mock-worker.test.ts rename to packages/engine-multi/test/worker/mock-worker.test.ts index f39db8b75..22e6521e3 100644 --- a/packages/engine-multi/test/mock-worker.test.ts +++ b/packages/engine-multi/test/worker/mock-worker.test.ts @@ -9,9 +9,9 @@ import path from 'node:path'; import test from 'ava'; import workerpool from 'workerpool'; -import { createPlan } from './util'; +import { createPlan } from '../../src/test/util'; -import * as e from '../src/events'; +import * as e from '../../src/events'; const workers = workerpool.pool(path.resolve('dist/mock-worker.js')); diff --git a/packages/engine-multi/test/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts similarity index 100% rename from packages/engine-multi/test/worker-pool.test.ts rename to packages/engine-multi/test/worker/worker-pool.test.ts From 8cd30088e91a6f78d5c4761b4517248aaf33cf0b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 12 Oct 2023 17:13:39 +0100 Subject: [PATCH 140/232] engine: add pack --- packages/engine-multi/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 065afcf1b..0badef4b4 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -10,7 +10,8 @@ "test:types": "pnpm tsc --noEmit --project tsconfig.json", "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting --transpile-only", "_build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", - "build:watch": "pnpm build --watch" + "build:watch": "pnpm build --watch", + "pack": "pnpm pack --pack-destination ../../dist" }, "author": "Open Function Group ", "license": "ISC", From de9d5975f0da1b5803695e627057a10605fc4302 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 10:17:52 +0100 Subject: [PATCH 141/232] engine: api tests, fix ava issue --- packages/engine-multi/package.json | 7 ++--- packages/engine-multi/src/api.ts | 10 ++----- packages/engine-multi/src/api/call-worker.ts | 10 ++++++- packages/engine-multi/src/api/execute.ts | 5 ++++ packages/engine-multi/src/engine.ts | 10 +++++-- packages/engine-multi/src/types.d.ts | 2 +- packages/engine-multi/src/worker/worker.ts | 7 +++-- packages/engine-multi/test.js | 29 +++++++++++++++++++ packages/engine-multi/test/api.test.ts | 9 ++++-- packages/engine-multi/test/engine.test.ts | 14 ++++----- .../engine-multi/test/integration.test.ts | 14 +++++++++ packages/engine-multi/tsup.config.js | 12 ++++++++ 12 files changed, 101 insertions(+), 28 deletions(-) create mode 100644 packages/engine-multi/test.js create mode 100644 packages/engine-multi/test/integration.test.ts create mode 100644 packages/engine-multi/tsup.config.js diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 0badef4b4..1d25db8a7 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -8,8 +8,7 @@ "scripts": { "test": "pnpm ava", "test:types": "pnpm tsc --noEmit --project tsconfig.json", - "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts src/worker.ts --no-splitting --transpile-only", - "_build": "tsup --config ../../tsup.config.js src/mock-worker.ts --no-splitting", + "build": "tsup --config ./tsup.config.js", "build:watch": "pnpm build --watch", "pack": "pnpm pack --pack-destination ../../dist" }, @@ -21,13 +20,13 @@ "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", "koa": "^2.13.4", - "workerpool": "^6.2.1" + "workerpool": "^6.5.1" }, "devDependencies": { "@types/koa": "^2.13.5", "@types/node": "^18.15.13", "@types/nodemon": "^1.19.2", - "@types/workerpool": "^6.1.0", + "@types/workerpool": "^6.4.4", "ava": "5.3.1", "nodemon": "^2.0.19", "ts-node": "^10.9.1", diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 8784704f4..0e4130d19 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -1,7 +1,6 @@ // Creates the public/external API to the runtime // Basically a thin wrapper, with validation, around the engine -import path from 'node:path'; import createLogger, { Logger } from '@openfn/logger'; import createEngine from './engine'; @@ -68,12 +67,9 @@ const createAPI = function (options: RTEOptions = {}) { autoinstall: options.autoinstall, }; - // Create the internal API - // TMP: use the mock worker for now - const engine = createEngine( - engineOptions, - path.resolve('dist/mock-worker.js') - ); + // Note that the engine here always uses the standard worker, the real one + // To use a mock, create the engine directly + const engine = createEngine(engineOptions); // Return the external API return { diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 79df475cd..2d895428d 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -35,5 +35,13 @@ export function createWorkers(workerPath: string) { resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); } - return workerpool.pool(resolvedWorkerPath); + return workerpool.pool(resolvedWorkerPath, { + workerThreadOpts: { + // Note that we have to pass this explicitly to run in ava's test runner + execArgv: ['--no-warnings', '--experimental-vm-modules'], + // // TODO if this unset, can the thread read the parent env? + // Also todo I think this hides experimental vm modules so it all breaks + // env: {}, + }, + }); } diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index d77204c11..fae72dc72 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -32,6 +32,11 @@ const execute = async (context: ExecutionContext) => { return callWorker('run', [state.plan, adaptorPaths], events).catch( (e: any) => { // TODO what about errors then? + + // If the worker file can't be found, we get: + // code: MODULE_NOT_FOUND + // message: cannot find modulle (worker.js) + logger.error(e); } ); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 6f0dcb4bd..a399f587e 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -97,6 +97,8 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { // TODO I think this is for later //const activeWorkflows: string[] = []; + // TOOD I wonder if the engine should a) always accept a worker path + // and b) validate it before it runs let resolvedWorkerPath; if (workerPath) { // If a path to the worker has been passed in, just use it verbatim @@ -105,7 +107,12 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { } else { // By default, we load ./worker.js but can't rely on the working dir to find it const dirname = path.dirname(fileURLToPath(import.meta.url)); - resolvedWorkerPath = path.resolve(dirname, workerPath || './worker.js'); + resolvedWorkerPath = path.resolve( + dirname, + // TODO there are too many assumptions here, it's an argument for the path just to be + // passed by the mian api or the unit test + workerPath || '../dist/worker/worker.js' + ); } options.logger!.debug('Loading workers from ', resolvedWorkerPath); @@ -165,7 +172,6 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { const listen = ( workflowId: string, - //handlers: Partial> handlers: Record ) => { const events = contexts[workflowId]; diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index cf0c25cb7..fc71271b7 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -90,7 +90,7 @@ export type WorkerLogPayload = { message: JSONLog; }; -type EventHandler = ( +export type EventHandler = ( event: EventPayloadLookup[T] ) => void; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 5bc2fa342..6b67d107b 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -1,6 +1,3 @@ -// Runs inside the worker -// TODO maybe this is - // Dedicated worker for running jobs // Security thoughts: the process inherits the node command arguments // (it has to for experimental modules to work) @@ -19,6 +16,10 @@ import type { ExecutionPlan } from '@openfn/runtime'; import helper, { createLoggers } from './worker-helper'; workerpool.worker({ + // TODO: add a startup script to ensure the worker is ok + // if we can't call init, there's something wrong with the worker + // and then we have to abort the engine or something + //init: () => {}, run: ( plan: ExecutionPlan, adaptorPaths: Record diff --git a/packages/engine-multi/test.js b/packages/engine-multi/test.js new file mode 100644 index 000000000..0ecb6918b --- /dev/null +++ b/packages/engine-multi/test.js @@ -0,0 +1,29 @@ +import createAPI from './dist/index.js'; +import createLogger from '@openfn/logger'; + +const api = createAPI({ + logger: createLogger(null, { level: 'debug' }), + // Disable compilation + compile: { + skip: true, + }, +}); + +const plan = { + id: 'a', + jobs: [ + { + expression: `export default [s => s]`, + // with no adaptor it shouldn't try to autoinstall + }, + ], +}; + +// this basically works so long as --experimental-vm-modules is on +// although the event doesn't feed through somehow, but that's different +const listener = api.execute(plan); +listener.on('workflow-complete', ({ state }) => { + console.log(state); + console.log('workflow completed'); + process.exit(0); +}); diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index 809de9de4..fa326379d 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -1,5 +1,4 @@ import test from 'ava'; - import createAPI from '../src/api'; import { createMockLogger } from '@openfn/logger'; @@ -32,6 +31,10 @@ test('create an engine api with a limited surface', (t) => { t.deepEqual(keys, ['execute', 'listen']); }); +// Note that this runs with the actual runtime worker +// I won't want to do deep testing on execute here - I just want to make sure the basic +// exeuction functionality is working. It's more a test of the api surface than the inner +// workings of the job test('execute should return an event listener and receive workflow-complete', (t) => { return new Promise((done) => { const api = createAPI({ @@ -46,7 +49,7 @@ test('execute should return an event listener and receive workflow-complete', (t id: 'a', jobs: [ { - expression: 's => s', + expression: 'export default [s => s]', // with no adaptor it shouldn't try to autoinstall }, ], @@ -74,7 +77,7 @@ test('should listen to workflow-complete', (t) => { id: 'a', jobs: [ { - expression: 's => s', + expression: 'export default [s => s]', // with no adaptor it shouldn't try to autoinstall }, ], diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 9cd7e1d3e..e158b808e 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -17,7 +17,7 @@ const options = { // Just not the runtime logic workerPath: path.resolve('dist/mock-worker.js'), logger, - repoDir: '', // doesn't matter for the mock + repoDir: '.', // doesn't matter for the mock noCompile: true, // messy - needed to allow an expression to be passed as json }; @@ -26,7 +26,7 @@ test.afterEach(() => { }); test('create an engine', (t) => { - const engine = createEngine({ logger }); + const engine = createEngine(options); t.truthy(engine); t.is(engine.constructor.name, 'Engine'); t.truthy(engine.execute); @@ -37,7 +37,7 @@ test('create an engine', (t) => { test('register a workflow', (t) => { const plan = { id: 'z' }; - const engine = createEngine({ logger }); + const engine = createEngine(options); const state = engine.registerWorkflow(plan); @@ -48,7 +48,7 @@ test('register a workflow', (t) => { test('get workflow state', (t) => { const plan = { id: 'z' } as ExecutionPlan; - const engine = createEngine({ logger }); + const engine = createEngine(options); const s = engine.registerWorkflow(plan); @@ -58,15 +58,15 @@ test('get workflow state', (t) => { }); test('use the default worker path', (t) => { - const engine = createEngine({ logger }); + const engine = createEngine({ logger, repoDir: '.' }); // this is how the path comes out in the test framework - t.true(engine.workerPath.endsWith('src/worker.js')); + t.true(engine.workerPath.endsWith('src/worker/worker.js')); }); // Note that even though this is a nonsense path, we get no error at this point test('use a custom worker path', (t) => { const p = 'jam'; - const engine = createEngine({ logger }, p); + const engine = createEngine(options, p); // this is how the path comes out in the test framework t.is(engine.workerPath, p); }); diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts new file mode 100644 index 000000000..94efe1ee7 --- /dev/null +++ b/packages/engine-multi/test/integration.test.ts @@ -0,0 +1,14 @@ +import test from 'ava'; +// this tests the full API with the actual runtime +// note that it won't test autoinstall +// (using jobs with no adaptors should be fine) + +// actually putting in a good suite of tests here and now is probably more valuable +// than "full" integration tests + +test.todo('should trigger workflow-start'); +test.todo('should trigger job-start'); +test.todo('should trigger job-complete'); +test.todo('should trigger multiple job-completes'); +test.todo('should trigger workflow-end'); +test.todo('should trigger workflow-logs'); diff --git a/packages/engine-multi/tsup.config.js b/packages/engine-multi/tsup.config.js new file mode 100644 index 000000000..8934555d4 --- /dev/null +++ b/packages/engine-multi/tsup.config.js @@ -0,0 +1,12 @@ +import baseConfig from '../../tsup.config'; + +export default { + ...baseConfig, + splitting: false, + entry: { + index: 'src/index.ts', + 'worker/worker': 'src/worker/worker.ts', + // TODO I don't actually want to build this into the dist, I don't think? + 'worker/mock': 'src/worker/mock-worker.ts', + }, +}; From f8fa02f1f625a49b76fb60a8e2c70fa870de7ada Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 13:07:29 +0100 Subject: [PATCH 142/232] engine: more tests, fix tests --- packages/engine-multi/src/api/lifecycle.ts | 3 +- .../engine-multi/src/worker/worker-helper.ts | 19 +- .../engine-multi/test/api/execute.test.ts | 2 +- .../engine-multi/test/api/lifecycle.test.ts | 2 +- packages/engine-multi/test/engine.test.ts | 4 +- .../engine-multi/test/integration.test.ts | 188 +++++++++++++++++- .../test/worker/mock-worker.test.ts | 2 +- 7 files changed, 203 insertions(+), 17 deletions(-) diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 2f789ae88..6fdb5ead3 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -36,6 +36,7 @@ export const workflowStart = ( // forward the event on to any external listeners context.emit(WORKFLOW_START, { + threadId, workflowId, // if this is a bespoke emitter it can be implied, which is nice // Should we publish anything else here? }); @@ -66,7 +67,7 @@ export const workflowComplete = ( // forward the event on to any external listeners context.emit(WORKFLOW_COMPLETE, { - workflowId: workflowId, + workflowId, threadId, duration: state.duration, state: result, diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts index c3fc49aee..e1826db7e 100644 --- a/packages/engine-multi/src/worker/worker-helper.ts +++ b/packages/engine-multi/src/worker/worker-helper.ts @@ -15,7 +15,7 @@ export const createLoggers = (workflowId: string) => { const log = (message: string) => { // hmm, the json log stringifies the message // i don't really want it to do that - publish({ + workerpool.workerEmit({ workflowId, type: e.WORKFLOW_LOG, message: JSON.parse(message), @@ -46,21 +46,32 @@ export const createLoggers = (workflowId: string) => { return { logger, jobLogger }; }; +// TODO use bespoke event names here +// maybe thread:workflow-start async function helper(workflowId: string, execute: () => Promise) { - publish({ type: e.WORKFLOW_START, workflowId, threadId }); + function publish(type: string, payload: any = {}) { + workerpool.workerEmit({ + workflowId, + threadId, + type, + ...payload, + } as e.WorkflowEvent); + } + + publish(e.WORKFLOW_START); try { // Note that the worker thread may fire logs after completion // I think this is fine, it's just a log stream thing // But the output is very confusing! const result = await execute(); - publish({ type: e.WORKFLOW_COMPLETE, workflowId, state: result }); + publish(e.WORKFLOW_COMPLETE, { state: result }); // For tests return result; } catch (err) { console.error(err); // @ts-ignore TODO sort out error typing - publish({ type: e.WORKFLOW_ERROR, workflowId, message: err.message }); + publish(e.WORKFLOW_ERROR, { message: err.message }); } } diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index 204537ef1..f9ed1127a 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -12,7 +12,7 @@ import { } from '../../src/events'; import { RTEOptions } from '../../src/api'; -const workerPath = path.resolve('dist/mock-worker.js'); +const workerPath = path.resolve('dist/worker/mock.js'); const createContext = ({ state, options }: Partial = {}) => { const api = new EventEmitter(); diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index f897b42a5..72cb24df2 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -23,7 +23,7 @@ test(`workflowStart: emits ${e.WORKFLOW_START}`, (t) => { const event = { workflowId, threadId: '123' }; context.on(e.WORKFLOW_START, (evt) => { - t.deepEqual(evt, { workflowId }); + t.deepEqual(evt, event); done(); }); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index e158b808e..a4c021ca9 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -59,15 +59,13 @@ test('get workflow state', (t) => { test('use the default worker path', (t) => { const engine = createEngine({ logger, repoDir: '.' }); - // this is how the path comes out in the test framework - t.true(engine.workerPath.endsWith('src/worker/worker.js')); + t.true(engine.workerPath.endsWith('worker/worker.js')); }); // Note that even though this is a nonsense path, we get no error at this point test('use a custom worker path', (t) => { const p = 'jam'; const engine = createEngine(options, p); - // this is how the path comes out in the test framework t.is(engine.workerPath, p); }); diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 94efe1ee7..4145b8803 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -1,4 +1,13 @@ import test from 'ava'; +import createAPI from '../src/api'; +import { createMockLogger } from '@openfn/logger'; + +const logger = createMockLogger(); + +test.afterEach(() => { + logger._reset(); +}); + // this tests the full API with the actual runtime // note that it won't test autoinstall // (using jobs with no adaptors should be fine) @@ -6,9 +15,176 @@ import test from 'ava'; // actually putting in a good suite of tests here and now is probably more valuable // than "full" integration tests -test.todo('should trigger workflow-start'); -test.todo('should trigger job-start'); -test.todo('should trigger job-complete'); -test.todo('should trigger multiple job-completes'); -test.todo('should trigger workflow-end'); -test.todo('should trigger workflow-logs'); +const withFn = `function fn(f) { return (s) => f(s) } +`; + +let idgen = 0; + +const createPlan = (jobs?: any[]) => ({ + id: `${++idgen}`, + jobs: jobs || [ + { + expression: 'export default [s => s]', + }, + ], +}); + +test('trigger workflow-start', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + compile: { + skip: true, + }, + }); + + const plan = createPlan(); + + api.execute(plan).on('workflow-start', (evt) => { + t.is(evt.workflowId, plan.id); + t.truthy(evt.threadId); + t.pass('workflow started'); + done(); + }); + }); +}); + +test.skip('trigger job-start', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + compile: { + skip: true, + }, + }); + + const plan = createPlan(); + + api.execute(plan).on('job-start', () => { + t.pass('job started'); + done(); + }); + }); +}); + +test.skip('trigger job-complete', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + compile: { + skip: true, + }, + }); + + const plan = createPlan(); + + api.execute(plan).on('job-complete', () => { + t.pass('job completed'); + done(); + }); + }); +}); + +test.todo('trigger multiple job-completes'); + +test('trigger workflow-complete', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + compile: { + skip: true, + }, + }); + + const plan = createPlan(); + + api.execute(plan).on('workflow-complete', (evt) => { + t.is(evt.workflowId, plan.id); + t.truthy(evt.duration); + t.truthy(evt.state); + t.truthy(evt.threadId); + t.pass('workflow completed'); + done(); + }); + }); +}); + +test('trigger workflow-log for job logs', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + compile: { + skip: true, + }, + }); + + const plan = createPlan([ + { + expression: `${withFn}console.log('hola')`, + }, + ]); + + api.execute(plan).on('workflow-log', (evt) => { + if (evt.name === 'JOB') { + t.deepEqual(evt.message, ['hola']); + t.pass('workflow logged'); + done(); + } + }); + }); +}); + +test('compile and run', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + }); + + const plan = createPlan([ + { + expression: `${withFn}fn(() => ({ data: 42 }))`, + }, + ]); + + api.execute(plan).on('workflow-complete', ({ state }) => { + t.deepEqual(state.data, 42); + done(); + }); + }); +}); + +test('evaluate conditional edges', (t) => { + return new Promise((done) => { + const api = createAPI({ + logger, + }); + + const jobs = [ + { + id: 'a', + next: { + b: true, + c: false, + }, + }, + { + id: 'b', + expression: `${withFn}fn(() => ({ data: 'b' }))`, + }, + { + id: 'c', + expression: `${withFn}fn(() => ({ data: 'c' }))`, + }, + ]; + + const plan = createPlan(jobs); + + api.execute(plan).on('workflow-complete', ({ state }) => { + t.deepEqual(state.data, 'b'); + done(); + }); + }); +}); + +test.todo('should report an error'); +test.todo('various workflow options (start, initial state)'); diff --git a/packages/engine-multi/test/worker/mock-worker.test.ts b/packages/engine-multi/test/worker/mock-worker.test.ts index 22e6521e3..662e5e3ab 100644 --- a/packages/engine-multi/test/worker/mock-worker.test.ts +++ b/packages/engine-multi/test/worker/mock-worker.test.ts @@ -13,7 +13,7 @@ import { createPlan } from '../../src/test/util'; import * as e from '../../src/events'; -const workers = workerpool.pool(path.resolve('dist/mock-worker.js')); +const workers = workerpool.pool(path.resolve('dist/worker/mock.js')); test('execute a mock plan inside a worker thread', async (t) => { const plan = createPlan(); From ccd4eda54b4f4235c1a4c378bf396e934104dbf1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 15:35:14 +0100 Subject: [PATCH 143/232] engine: split internal and external events --- packages/engine-multi/src/api/execute.ts | 15 ++-- packages/engine-multi/src/api/lifecycle.ts | 28 +++--- packages/engine-multi/src/engine.ts | 1 + packages/engine-multi/src/events.ts | 77 +++++++++++----- .../engine-multi/src/test/worker-functions.js | 6 +- packages/engine-multi/src/types.d.ts | 90 ++----------------- packages/engine-multi/src/worker/events.ts | 59 ++++++++++++ .../engine-multi/src/worker/worker-helper.ts | 31 ++++--- .../engine-multi/test/api/execute.test.ts | 2 +- .../engine-multi/test/api/lifecycle.test.ts | 2 + packages/engine-multi/test/engine.test.ts | 1 + .../test/worker/mock-worker.test.ts | 4 +- 12 files changed, 162 insertions(+), 154 deletions(-) create mode 100644 packages/engine-multi/src/worker/events.ts diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index fae72dc72..2ced96a48 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -1,15 +1,11 @@ // Execute a compiled workflow -import * as e from '../events'; +import * as workerEvents from '../worker/events'; import { ExecutionContext } from '../types'; import autoinstall from './autoinstall'; import compile from './compile'; import { workflowStart, workflowComplete, log } from './lifecycle'; -// A lot of callbacks needed here -// Is it better to just return the handler? -// But then this function really isn't doing so much -// (I guess that's true anyway) const execute = async (context: ExecutionContext) => { const { state, callWorker, logger } = context; @@ -17,14 +13,15 @@ const execute = async (context: ExecutionContext) => { await compile(context); const events = { - // TODO typings - [e.WORKFLOW_START]: (evt: any) => { + [workerEvents.WORKFLOW_START]: (evt: workerEvents.WorkflowStartEvent) => { workflowStart(context, evt); }, - [e.WORKFLOW_COMPLETE]: (evt: any) => { + [workerEvents.WORKFLOW_COMPLETE]: ( + evt: workerEvents.WorkflowCompleteEvent + ) => { workflowComplete(context, evt); }, - [e.WORKFLOW_LOG]: (evt: any) => { + [workerEvents.LOG]: (evt: workerEvents.LogEvent) => { log(context, evt); }, }; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 6fdb5ead3..151235ed5 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -1,14 +1,11 @@ -import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from '../events'; -import { - ExecutionContext, - WorkerCompletePayload, - WorkerLogPayload, - WorkerStartPayload, -} from '../types'; +// here's here things get a bit complex event wise +import * as externalEvents from '../events'; +import * as internalEvents from '../worker/events'; +import { ExecutionContext } from '../types'; export const workflowStart = ( context: ExecutionContext, - event: WorkerStartPayload // the event published by the runtime itself ({ workflowId, threadId }) + event: internalEvents.WorkflowStartEvent ) => { const { state, logger } = context; const { workflowId, threadId } = event; @@ -35,7 +32,7 @@ export const workflowStart = ( // api.activeWorkflows.push(workflowId); // forward the event on to any external listeners - context.emit(WORKFLOW_START, { + context.emit(externalEvents.WORKFLOW_START, { threadId, workflowId, // if this is a bespoke emitter it can be implied, which is nice // Should we publish anything else here? @@ -44,7 +41,7 @@ export const workflowStart = ( export const workflowComplete = ( context: ExecutionContext, - event: WorkerCompletePayload // the event published by the runtime itself ({ workflowId, threadId }) + event: internalEvents.WorkflowCompleteEvent ) => { const { logger, state } = context; const { workflowId, state: result, threadId } = event; @@ -66,7 +63,7 @@ export const workflowComplete = ( // activeWorkflows.splice(idx, 1); // forward the event on to any external listeners - context.emit(WORKFLOW_COMPLETE, { + context.emit(externalEvents.WORKFLOW_COMPLETE, { workflowId, threadId, duration: state.duration, @@ -76,9 +73,9 @@ export const workflowComplete = ( export const log = ( context: ExecutionContext, - event: WorkerLogPayload // the event published by the runtime itself ({ workflowId, threadId }) + event: internalEvents.LogEvent ) => { - const { id } = context.state; + const { workflowId, threadId } = event; // // TODO not sure about this stuff, I think we can drop it? // const newMessage = { // ...message, @@ -88,8 +85,9 @@ export const log = ( // }; context.logger.proxy(event.message); - context.emit(WORKFLOW_LOG, { - workflowId: id, + context.emit(externalEvents.WORKFLOW_LOG, { + workflowId, + threadId, ...event.message, }); }; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index a399f587e..71e684f6a 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -83,6 +83,7 @@ export type EngineOptions = { resolvers?: LazyResolvers; + noCompile?: boolean; compile?: {}; // TODO autoinstall?: AutoinstallOptions; }; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 86aa31f70..d67b25e72 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -12,39 +12,70 @@ export const WORKFLOW_COMPLETE = 'workflow-complete'; export const WORKFLOW_ERROR = 'workflow-error'; +export const JOB_START = 'job-start'; + +export const JOB_COMPLETE = 'job-complete'; + export const WORKFLOW_LOG = 'workflow-log'; -// Internal runtime events - these are what the worker thread publishes -// to the engine +export const WORKFLOW_EDGE_RESOLVED = 'workflow-edge-resolved'; + +export type EventMap = { + [WORKFLOW_START]: WorkflowStartPayload; + [WORKFLOW_COMPLETE]: WorkflowCompletePayload; + [JOB_START]: JobStartPayload; + [JOB_COMPLETE]: JobCompletePayload; + [WORKFLOW_LOG]: WorkerLogPayload; + [WORKFLOW_ERROR]: WorkflowErrorPayload; +}; + +export type ExternalEvents = keyof EventMap; -type State = any; // TODO +interface ExternalEvent { + threadId: string; + workflowId: string; +} -export type AcceptWorkflowEvent = { - type: typeof WORKFLOW_START; +export interface WorkflowStartPayload extends ExternalEvent { + threadId: string; workflowId: string; - threadId: number; -}; +} -export type CompleteWorkflowEvent = { - type: typeof WORKFLOW_COMPLETE; +export interface WorkflowCompletePayload extends ExternalEvent { + threadId: string; workflowId: string; - state: State; -}; + state: any; + duration: number; +} -export type ErrWorkflowEvent = { - type: typeof WORKFLOW_ERROR; +export interface WorkflowErrorPayload extends ExternalEvent { + threadId: string; workflowId: string; + type: string; message: string; -}; +} -export type LogWorkflowEvent = { - type: typeof WORKFLOW_LOG; +export interface JobStartPayload extends ExternalEvent { + threadId: string; workflowId: string; - message: JSONLog; -}; + jobId: string; +} + +export interface JobCompletePayload extends ExternalEvent { + threadId: string; + workflowId: string; + jobId: string; + state: any; // the result state +} -export type WorkflowEvent = - | AcceptWorkflowEvent - | CompleteWorkflowEvent - | ErrWorkflowEvent - | LogWorkflowEvent; +export interface WorkerLogPayload extends ExternalEvent, JSONLog { + threadId: string; + workflowId: string; +} + +export interface EdgeResolvedPayload extends ExternalEvent { + threadId: string; + workflowId: string; + edgeId: string; // interesting, we don't really have this yet. Is index more appropriate? key? yeah, it's target node basically + result: boolean; +} diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index c4e1221e3..7ba254136 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -19,7 +19,7 @@ workerpool.worker({ run: (plan, adaptorPaths) => { const workflowId = plan.id; workerpool.workerEmit({ - type: 'workflow-start', + type: 'worker:workflow-start', workflowId, threadId, }); @@ -28,7 +28,7 @@ workerpool.worker({ const result = eval(job.expression); workerpool.workerEmit({ - type: 'workflow-complete', + type: 'worker:workflow-complete', workflowId, state: result, threadId, @@ -37,7 +37,7 @@ workerpool.worker({ console.error(err); // @ts-ignore TODO sort out error typing workerpool.workerEmit({ - type: 'workflow-error', + type: 'worker:workflow-error', workflowId, message: err.message, threadId, diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index fc71271b7..643bd3db7 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -5,92 +5,10 @@ import type { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; import { RTEOptions } from './api'; -// These are the external events published but he api and listen -type WorkflowStartEvent = 'workflow-start'; -type WorkflowStartPayload = { - workflowId: string; -}; - -type WorkflowCompleteEvent = 'workflow-complete'; -type WorkflowCompletePayload = { - workflowId: string; -}; - -// This occurs when a critical error causes the workflow to be aborted -// ie crash, syntax error -type WorkflowErrorEvent = 'workflow-error'; -type WorkflowErrorPayload = { - workflowId: string; - type: string; - message: string; -}; - -type JobStartEvent = 'job-start'; -type JobStartPayload = { - workflowId: string; - jobId: string; -}; - -type JobCompleteEvent = 'job-complete'; -type JobCompletePayload = { - workflowId: string; - jobId: string; - state: any; // the result start -}; - -// a log message coming out of the engine, adaptor or job -type LogEvent = 'job-complete'; -type LogPayload = JSONLog & { - workflowId: string; -}; - -// TODO -type EdgeResolvedEvent = 'edge-resolved'; -type EdgeResolvedPayload = { - workflowId: string; - edgeId: string; // interesting, we don't really have this yet. Is index more appropriate? key? yeah, it's target node basically - result: boolean; -}; - -type EngineEvents = - | WorkflowStartEvent - | WorkflowCompleteEvent - | JobStartEvent - | JobCompleteEvent - | LogEvent; - -type EventPayloadLookup = { - [WorkflowStartEvent]: WorkflowStartPayload; - [WorkflowCompleteEvent]: WorkflowCompletePayload; - [JobStartEvent]: JobStartPayload; - [JobCompleteEvent]: JobCompletePayload; - [LogEvent]: LogPayload; -}; - -// These are events from the internal worker (& runtime) - -export type WORKER_START = 'worker-start'; -export type WorkerStartPayload = { - threadId: string; - workflowId: string; -}; - -export type WORKER_COMPLETE = 'worker-complete'; -export type WorkerCompletePayload = { - threadId: string; - workflowId: string; - state: any; -}; +import { ExternalEvents, EventMap } from './events'; -// TODO confusion over this and events.ts -export type WORKER_LOG = 'worker-log'; -export type WorkerLogPayload = { - threadId: string; - workflowId: string; - message: JSONLog; -}; - -export type EventHandler = ( +// TODO hmm, not sure about this - event handler for what? +export type EventHandler = ( event: EventPayloadLookup[T] ) => void; @@ -138,6 +56,8 @@ export interface ExecutionContext extends EventEmitter { state: WorkflowState; logger: Logger; callWorker: CallWorker; + + emit(event: T, payload: EventMap[T]): boolean; } export interface EngineAPI extends EventEmitter { diff --git a/packages/engine-multi/src/worker/events.ts b/packages/engine-multi/src/worker/events.ts new file mode 100644 index 000000000..0e3f4b3bc --- /dev/null +++ b/packages/engine-multi/src/worker/events.ts @@ -0,0 +1,59 @@ +/** + * Events published by the inner worker + */ + +import { JSONLog } from '@openfn/logger'; + +// These events are basically duplicates of the externally published ones +// (ie those consumed by the lightning worker) +// But I want them to be explicity named and typed to avoid confusion +// Otherwise when you're looking an event, it's hard to know if it's internal +// or external + +export const WORKFLOW_START = 'worker:workflow-start'; + +export const WORKFLOW_COMPLETE = 'worker:workflow-complete'; + +export const JOB_START = 'worker:job-start'; + +export const JOB_COMPLETE = 'worker:job-complete'; + +export const ERROR = 'worker:error'; + +export const LOG = 'worker:log'; + +interface InternalEvent { + type: WorkerEvents; + workflowId: string; + threadId: string; +} + +export interface WorkflowStartEvent extends InternalEvent {} + +export interface WorkflowCompleteEvent extends InternalEvent { + state: any; +} + +export interface JobStartEvent extends InternalEvent {} + +export interface JobCompleteEvent extends InternalEvent {} + +export interface LogEvent extends InternalEvent { + message: JSONLog; +} + +// TODO not really sure what errors look like yet +export interface ErrorEvent extends InternalEvent { + [key: string]: any; +} + +export type EventMap = { + [WORKFLOW_START]: WorkflowStartEvent; + [WORKFLOW_COMPLETE]: WorkflowCompleteEvent; + [JOB_START]: JobStartEvent; + [JOB_COMPLETE]: JobCompleteEvent; + [LOG]: LogEvent; + [ERROR]: ErrorEvent; +}; + +export type WorkerEvents = keyof EventMap; diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts index e1826db7e..852619cc8 100644 --- a/packages/engine-multi/src/worker/worker-helper.ts +++ b/packages/engine-multi/src/worker/worker-helper.ts @@ -5,21 +5,17 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; import createLogger from '@openfn/logger'; -import * as e from '../events'; - -function publish(event: e.WorkflowEvent) { - workerpool.workerEmit(event); -} +import * as workerEvents from './events'; export const createLoggers = (workflowId: string) => { const log = (message: string) => { - // hmm, the json log stringifies the message - // i don't really want it to do that + // Apparently the json log stringifies the message + // We don't really want it to do that workerpool.workerEmit({ workflowId, - type: e.WORKFLOW_LOG, + type: workerEvents.LOG, message: JSON.parse(message), - }); + } as workerEvents.LogEvent); }; const emitter: any = { @@ -49,29 +45,32 @@ export const createLoggers = (workflowId: string) => { // TODO use bespoke event names here // maybe thread:workflow-start async function helper(workflowId: string, execute: () => Promise) { - function publish(type: string, payload: any = {}) { + function publish( + type: T, + payload: Omit + ) { workerpool.workerEmit({ workflowId, threadId, type, ...payload, - } as e.WorkflowEvent); + }); } - publish(e.WORKFLOW_START); + publish(workerEvents.WORKFLOW_START, {}); + try { // Note that the worker thread may fire logs after completion // I think this is fine, it's just a log stream thing // But the output is very confusing! const result = await execute(); - publish(e.WORKFLOW_COMPLETE, { state: result }); + publish(workerEvents.WORKFLOW_COMPLETE, { state: result }); // For tests return result; - } catch (err) { + } catch (err: any) { console.error(err); - // @ts-ignore TODO sort out error typing - publish(e.WORKFLOW_ERROR, { message: err.message }); + publish(workerEvents.ERROR, { workflowId, threadId, message: err.message }); } } diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index f9ed1127a..f01f89b7a 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -88,7 +88,7 @@ test.serial('should emit a workflow-complete event', async (t) => { t.is(workflowComplete.state, 22); }); -test.serial('should emit a log event', async (t) => { +test.serial.only('should emit a log event', async (t) => { let workflowLog; const plan = { id: 'y', diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 72cb24df2..ffb0b4fc3 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -100,6 +100,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { const event = { workflowId, + threadId: 'a', message: { level: 'info', name: 'job', @@ -111,6 +112,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { context.on(e.WORKFLOW_LOG, (evt) => { t.deepEqual(evt, { workflowId: state.id, + threadId: 'a', ...event.message, }); done(); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index a4c021ca9..a98c54fa2 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -75,6 +75,7 @@ test('execute with test worker and trigger workflow-complete', (t) => { const engine = createEngine( { logger, + repoDir: '.', noCompile: true, autoinstall: { handleIsInstalled: async () => true, diff --git a/packages/engine-multi/test/worker/mock-worker.test.ts b/packages/engine-multi/test/worker/mock-worker.test.ts index 662e5e3ab..a611a167b 100644 --- a/packages/engine-multi/test/worker/mock-worker.test.ts +++ b/packages/engine-multi/test/worker/mock-worker.test.ts @@ -11,7 +11,7 @@ import workerpool from 'workerpool'; import { createPlan } from '../../src/test/util'; -import * as e from '../../src/events'; +import * as e from '../../src/worker/events'; const workers = workerpool.pool(path.resolve('dist/worker/mock.js')); @@ -135,7 +135,7 @@ test('Publish a job log event', async (t) => { let id; await workers.exec('run', [plan], { on: ({ workflowId, type, message }) => { - if (type === e.WORKFLOW_LOG) { + if (type === e.LOG) { didFire = true; log = message; id = workflowId; From 195f0984b74f7c2fe02dd48825604b2aa1945c62 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 16:21:10 +0100 Subject: [PATCH 144/232] runtime: add callbacks for job execution --- .changeset/tough-coats-unite.md | 5 ++ packages/runtime/src/execute/expression.ts | 18 +++- packages/runtime/src/runtime.ts | 10 ++- packages/runtime/src/types.ts | 12 +++ .../runtime/test/execute/expression.test.ts | 83 ++++++++++++++++++- packages/runtime/test/runtime.test.ts | 69 +++++++++++++++ 6 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 .changeset/tough-coats-unite.md diff --git a/.changeset/tough-coats-unite.md b/.changeset/tough-coats-unite.md new file mode 100644 index 000000000..860937bde --- /dev/null +++ b/.changeset/tough-coats-unite.md @@ -0,0 +1,5 @@ +--- +'@openfn/runtime': patch +--- + +Trigger callbacks on job start, complete and init diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts index 7ac7a49e1..be363b71d 100644 --- a/packages/runtime/src/execute/expression.ts +++ b/packages/runtime/src/execute/expression.ts @@ -1,7 +1,7 @@ import { printDuration, Logger } from '@openfn/logger'; import stringify from 'fast-safe-stringify'; import loadModule from '../modules/module-loader'; -import { Operation, JobModule, State } from '../types'; +import { Operation, JobModule, State, ExecutionCallbacks } from '../types'; import { Options, ERR_TIMEOUT, TIMEOUT } from '../runtime'; import buildContext, { Context } from './context'; import defaultExecute from '../util/execute'; @@ -14,10 +14,15 @@ export default ( opts: Options = {} ) => new Promise(async (resolve, reject) => { + const { callbacks = {} } = opts; const timeout = opts.timeout || TIMEOUT; + logger.debug('Intialising pipeline'); logger.debug(`Timeout set to ${timeout}ms`); + let initDuration = Date.now(); + callbacks.onInitStart?.(); + // Setup an execution context const context = buildContext(initialState, opts); @@ -28,10 +33,15 @@ export default ( wrapOperation(op, logger, `${idx + 1}`, opts.immutableState) ) ); + initDuration = Date.now() - initDuration; + + callbacks.onInitComplete?.({ duration: initDuration }); // Run the pipeline try { logger.debug(`Executing expression (${operations.length} operations)`); + let exeDuration = Date.now(); + callbacks.onStart?.(); const tid = setTimeout(() => { logger.error(`Error: Timeout (${timeout}ms) expired!`); @@ -46,6 +56,10 @@ export default ( logger.debug('Expression complete!'); logger.debug(result); + exeDuration = Date.now() - exeDuration; + + callbacks.onComplete?.({ duration: exeDuration, state: result }); + // return the final state resolve(prepareFinalState(opts, result)); } catch (e: any) { @@ -76,6 +90,8 @@ const prepareJob = async ( context: Context, opts: Options = {} ): Promise => { + // TODO resolve credential + // TODO resolve initial state if (typeof expression === 'string') { const exports = await loadModule(expression, { ...opts.linker, diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index d00010901..47e18085b 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -1,6 +1,12 @@ import { createMockLogger, Logger } from '@openfn/logger'; -import type { Operation, ExecutionPlan, State, JobNodeID } from './types'; +import type { + Operation, + ExecutionPlan, + State, + JobNodeID, + ExecutionCallbacks, +} from './types'; import type { LinkerOptions } from './modules/linker'; import executePlan from './execute/plan'; import clone from './util/clone'; @@ -27,6 +33,8 @@ export type Options = { forceSandbox?: boolean; linker?: LinkerOptions; + + callbacks?: ExecutionCallbacks; }; const defaultState = { data: {}, configuration: {} }; diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 8d8545994..ed0fc7107 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -95,3 +95,15 @@ export type JobModule = { execute?: (...operations: Operation[]) => (state: any) => any; // TODO lifecycle hooks }; + +// I'm not wild about the callbacks pattern, events would be simpler +// but a) the resolvers MUST be a callback and b) we don't currently have an event emitter API +export type ExecutionCallbacks = { + onInitStart?: () => void; + onInitComplete?: (args: { duration: number }) => void; + onStart?: () => void; + onComplete?: (args: { duration: number; state: any }) => void; + + resolveState?: (state: string) => any; + resolveCredential?: (state: string) => any; +}; diff --git a/packages/runtime/test/execute/expression.test.ts b/packages/runtime/test/execute/expression.test.ts index 02fa12c6f..bbb8ffbf1 100644 --- a/packages/runtime/test/execute/expression.test.ts +++ b/packages/runtime/test/execute/expression.test.ts @@ -48,13 +48,94 @@ test('run a stringified no-op job with one operation', async (t) => { }); test('run a live no-op job with @openfn/language-common.fn', async (t) => { - const job = [fn((s: State) => s)] as Operation[]; + // @ts-ignore + const job = [fn((s) => s)]; const state = createState(); const result = await executeExpression(job, state); t.deepEqual(state, result); }); +// TODO all these need implementing +// do they happen in expression.ts or elsewhere? +// I think this is fine + +// on-start is called before things are resolved +// it means "i'm about to start processing this job" +// does duration include state loading? +// it's tricky, from the user's point of view it should just be execution time +// ok, fine, we're gonna say that start is literally just execution +// but we'll add like a job-initialise event, with a duration +test('call the on-intialise callback', async (t) => { + let didCallCallback = false; + + const job = [(s: State) => s]; + const state = createState(); + + const callbacks = { + onInitStart: () => { + didCallCallback = true; + }, + }; + + await executeExpression(job, state, { callbacks }); + t.true(didCallCallback); +}); + +test('call the on-intialise-complete callback', async (t) => { + let didCallCallback = false; + + const job = [(s: State) => s]; + const state = createState(); + + const callbacks = { + onInitComplete: ({ duration }: any) => { + t.assert(!isNaN(duration)); + didCallCallback = true; + }, + }; + + await executeExpression(job, state, { callbacks }); + t.true(didCallCallback); +}); + +test('call the on-start callback', async (t) => { + let didCallCallback = false; + + const job = [(s: State) => s]; + const state = createState(); + + const callbacks = { + onStart: () => { + didCallCallback = true; + }, + }; + + await executeExpression(job, state, { callbacks }); + t.true(didCallCallback); +}); + +test('call the on-complete callback', async (t) => { + let didCallCallback = false; + + const job = [(s: State) => s]; + const state = createState(); + + const callbacks = { + onComplete: ({ duration, state }: any) => { + t.assert(!isNaN(duration)); + t.truthy(state); + didCallCallback = true; + }, + }; + + await executeExpression(job, state, { callbacks }); + t.true(didCallCallback); +}); + +test.todo('resolve a credential'); +test.todo('resolve starting state'); + test('jobs can handle a promise', async (t) => { const job = [async (s: State) => s]; const state = createState(); diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 9315d5a39..e5793d13f 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -23,6 +23,75 @@ test('run a simple workflow', async (t) => { t.true(result.data.done); }); +test('run a workflow and call callbacks', async (t) => { + const counts: Record = {}; + const increment = (name: string) => { + return { + [name]: () => { + if (!counts[name]) { + counts[name] = 0; + } + counts[name] += 1; + }, + }; + }; + + const callbacks = Object.assign( + {}, + increment('onInitStart'), + increment('onInitComplete'), + increment('onStart'), + increment('onComplete') + ); + + const plan: ExecutionPlan = { + jobs: [{ expression: 'export default [(s) => s]' }], + }; + + await run(plan, {}, { callbacks }); + + t.is(counts.onInitStart, 1); + t.is(counts.onInitComplete, 1); + t.is(counts.onStart, 1); + t.is(counts.onComplete, 1); +}); + +test('run a workflow with two jobs and call callbacks', async (t) => { + const counts: Record = {}; + const increment = (name: string) => { + return { + [name]: () => { + if (!counts[name]) { + counts[name] = 0; + } + counts[name] += 1; + }, + }; + }; + + const callbacks = Object.assign( + {}, + increment('onInitStart'), + increment('onInitComplete'), + increment('onStart'), + increment('onComplete') + ); + + const plan: ExecutionPlan = { + jobs: [ + { id: 'a', expression: 'export default [(s) => s]', next: { b: true } }, + { id: 'b', expression: 'export default [(s) => s]' }, + ], + }; + + await run(plan, {}, { callbacks }); + + t.is(counts.onInitStart, 2); + t.is(counts.onInitComplete, 2); + t.is(counts.onStart, 2); + t.is(counts.onComplete, 2); +}); + test('run a workflow with state and parallel branching', async (t) => { const plan: ExecutionPlan = { jobs: [ From 99435b6677894357eac0df888105cb4fb6123d42 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 17:43:27 +0100 Subject: [PATCH 145/232] runtime: introduce a singly notify callback --- packages/runtime/src/execute/expression.ts | 33 ++- packages/runtime/src/execute/job.ts | 55 +++++ packages/runtime/src/execute/plan.ts | 66 +----- packages/runtime/src/types.ts | 27 ++- packages/runtime/src/util/notify.ts | 5 + .../runtime/test/execute/expression.test.ts | 207 ++++++++++-------- packages/runtime/test/runtime.test.ts | 66 +++--- 7 files changed, 247 insertions(+), 212 deletions(-) create mode 100644 packages/runtime/src/execute/job.ts create mode 100644 packages/runtime/src/util/notify.ts diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts index be363b71d..9633bd05c 100644 --- a/packages/runtime/src/execute/expression.ts +++ b/packages/runtime/src/execute/expression.ts @@ -1,27 +1,33 @@ import { printDuration, Logger } from '@openfn/logger'; import stringify from 'fast-safe-stringify'; import loadModule from '../modules/module-loader'; -import { Operation, JobModule, State, ExecutionCallbacks } from '../types'; +import { + Operation, + JobModule, + State, + ExecutionCallbacks, + ExecutionContext, +} from '../types'; import { Options, ERR_TIMEOUT, TIMEOUT } from '../runtime'; import buildContext, { Context } from './context'; import defaultExecute from '../util/execute'; import clone from '../util/clone'; export default ( + ctx: ExecutionContext, expression: string | Operation[], - initialState: State, - logger: Logger, - opts: Options = {} + initialState: State ) => new Promise(async (resolve, reject) => { - const { callbacks = {} } = opts; + const { logger, notify = () => {}, opts = {} } = ctx; const timeout = opts.timeout || TIMEOUT; logger.debug('Intialising pipeline'); logger.debug(`Timeout set to ${timeout}ms`); - let initDuration = Date.now(); - callbacks.onInitStart?.(); + // let initDuration = Date.now(); + + // callbacks.onInitStart?.(); // Setup an execution context const context = buildContext(initialState, opts); @@ -33,15 +39,17 @@ export default ( wrapOperation(op, logger, `${idx + 1}`, opts.immutableState) ) ); - initDuration = Date.now() - initDuration; + // initDuration = Date.now() - initDuration; - callbacks.onInitComplete?.({ duration: initDuration }); + // TODO actually we're going to do the init stuff in job.ts + // But I will notify() for how long it takes to load the job module + // callbacks.onInitComplete?.({ duration: initDuration }); // Run the pipeline try { logger.debug(`Executing expression (${operations.length} operations)`); let exeDuration = Date.now(); - callbacks.onStart?.(); + notify('job-start'); const tid = setTimeout(() => { logger.error(`Error: Timeout (${timeout}ms) expired!`); @@ -58,7 +66,7 @@ export default ( exeDuration = Date.now() - exeDuration; - callbacks.onComplete?.({ duration: exeDuration, state: result }); + notify('job-complete', { duration: exeDuration, state: result }); // return the final state resolve(prepareFinalState(opts, result)); @@ -92,6 +100,9 @@ const prepareJob = async ( ): Promise => { // TODO resolve credential // TODO resolve initial state + + // difficulty here that credential isn't passed in + if (typeof expression === 'string') { const exports = await loadModule(expression, { ...opts.linker, diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts new file mode 100644 index 000000000..869a6ba9e --- /dev/null +++ b/packages/runtime/src/execute/job.ts @@ -0,0 +1,55 @@ +// TODO hmm. I have a horrible feeling that the callbacks should go here +// at least the resolvesrs +import executeExpression from './expression'; +import type { + CompiledJobNode, + ExecutionContext, + JobNodeID, + State, +} from '../types'; + +// The job handler is responsible for preparing the job +// and working out where to go next +// it'll resolve credentials and state and notify how long init took +const executeJob = async ( + ctx: ExecutionContext, + job: CompiledJobNode, + state: State +): Promise<{ next: JobNodeID[]; state: any }> => { + const next: string[] = []; + + // We should by this point have validated the plan, so the job MUST exist + + ctx.logger.timer('job'); + ctx.logger.always('Starting job', job.id); + + let result: any = state; + if (job.expression) { + // The expression SHOULD return state, but could return anything + try { + result = await executeExpression(ctx, job.expression, state); + const duration = ctx.logger.timer('job'); + ctx.logger.success(`Completed job ${job.id} in ${duration}`); + } catch (e: any) { + const duration = ctx.logger.timer('job'); + ctx.logger.error(`Failed job ${job.id} after ${duration}`); + ctx.report(state, job.id, e); + } + } + + if (job.next) { + for (const nextJobId in job.next) { + const edge = job.next[nextJobId]; + if ( + edge && + (edge === true || !edge.condition || edge.condition(result)) + ) { + next.push(nextJobId); + } + // TODO errors + } + } + return { next, state: result }; +}; + +export default executeJob; diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index be1163e70..fe85fa370 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -1,25 +1,12 @@ import type { Logger } from '@openfn/logger'; -import executeExpression from './expression'; +import executeJob from './job'; import compilePlan from './compile-plan'; import assembleState from '../util/assemble-state'; -import type { - CompiledExecutionPlan, - CompiledJobNode, - ExecutionPlan, - JobNodeID, - State, -} from '../types'; +import type { ExecutionPlan, State } from '../types'; import type { Options } from '../runtime'; import clone from '../util/clone'; import validatePlan from '../util/validate-plan'; -import createErrorReporter, { ErrorReporter } from '../util/log-error'; - -type ExeContext = { - plan: CompiledExecutionPlan; - logger: Logger; - opts: Options; - report: ErrorReporter; -}; +import createErrorReporter from '../util/log-error'; const executePlan = async ( plan: ExecutionPlan, @@ -43,6 +30,7 @@ const executePlan = async ( opts, logger, report: createErrorReporter(logger), + notify: opts.callbacks?.notify, }; type State = any; @@ -84,50 +72,4 @@ const executePlan = async ( return Object.values(leaves)[0]; }; -const executeJob = async ( - ctx: ExeContext, - job: CompiledJobNode, - state: State -): Promise<{ next: JobNodeID[]; state: any }> => { - const next: string[] = []; - - // We should by this point have validated the plan, so the job MUST exist - - ctx.logger.timer('job'); - ctx.logger.always('Starting job', job.id); - - let result: any = state; - if (job.expression) { - // The expression SHOULD return state, but could return anything - try { - result = await executeExpression( - job.expression, - state, - ctx.logger, - ctx.opts - ); - const duration = ctx.logger.timer('job'); - ctx.logger.success(`Completed job ${job.id} in ${duration}`); - } catch (e: any) { - const duration = ctx.logger.timer('job'); - ctx.logger.error(`Failed job ${job.id} after ${duration}`); - ctx.report(state, job.id, e); - } - } - - if (job.next) { - for (const nextJobId in job.next) { - const edge = job.next[nextJobId]; - if ( - edge && - (edge === true || !edge.condition || edge.condition(result)) - ) { - next.push(nextJobId); - } - // TODO errors - } - } - return { next, state: result }; -}; - export default executePlan; diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index ed0fc7107..a4df7578f 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -1,4 +1,9 @@ // TMP just thinking through things + +import { Logger } from '@openfn/logger'; +import { Options } from './runtime'; +import { ErrorReporter } from './util/log-error'; + // I dont think this is useufl? We can just use error.name of the error object export type ErrorTypes = | 'AdaptorNotFound' // probably a CLI validation thing @@ -96,14 +101,26 @@ export type JobModule = { // TODO lifecycle hooks }; +// TODO difficulty: this is not the same as a vm execution context +export type ExecutionContext = { + plan: CompiledExecutionPlan; + logger: Logger; + opts: Options; + report: ErrorReporter; + notify: (evt: NotifyEvents, payload?: any) => void; +}; + +export type NotifyEvents = + | 'init-start' + | 'init-complete' + | 'job-start' + | 'job-complete'; + // I'm not wild about the callbacks pattern, events would be simpler // but a) the resolvers MUST be a callback and b) we don't currently have an event emitter API +// no, ok, we're going to have a notify function which does the callbacks export type ExecutionCallbacks = { - onInitStart?: () => void; - onInitComplete?: (args: { duration: number }) => void; - onStart?: () => void; - onComplete?: (args: { duration: number; state: any }) => void; - + notify(event: NotifyEvents, payload: any): void; resolveState?: (state: string) => any; resolveCredential?: (state: string) => any; }; diff --git a/packages/runtime/src/util/notify.ts b/packages/runtime/src/util/notify.ts new file mode 100644 index 000000000..cc4043edd --- /dev/null +++ b/packages/runtime/src/util/notify.ts @@ -0,0 +1,5 @@ +function createNotifier(callback) { + return (evt, payload) => { + callback(evt, payload); + }; +} diff --git a/packages/runtime/test/execute/expression.test.ts b/packages/runtime/test/execute/expression.test.ts index bbb8ffbf1..56f211688 100644 --- a/packages/runtime/test/execute/expression.test.ts +++ b/packages/runtime/test/execute/expression.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import { fn } from '@openfn/language-common'; -import type { State, Operation } from '../../src/types'; +import type { State, Operation, ExecutionContext } from '../../src/types'; import { createMockLogger } from '@openfn/logger'; import execute from '../../src/execute/expression'; @@ -17,11 +17,21 @@ const createState = (data = {}) => ({ const logger = createMockLogger(undefined, { level: 'debug' }); -const executeExpression = ( - job: string | Operation[], - state: State, - opts = {} -) => execute(job, state, logger, opts); +const createContext = (args = {}) => + ({ + logger, + plan: {}, + opts: {}, + notify: () => {}, + report: () => {}, + ...args, + } as unknown as ExecutionContext); + +// const executeExpression = ( +// job: string | Operation[], +// state: State, +// opts = {} +// ) => execute(job, state, logger, opts); test.afterEach(() => { logger._reset(); @@ -34,7 +44,10 @@ test.afterEach(() => { test('run a live no-op job with one operation', async (t) => { const job = [(s: State) => s]; const state = createState(); - const result = await executeExpression(job, state); + + const context = createContext(); + + const result = await execute(context, job, state); t.deepEqual(state, result); }); @@ -42,16 +55,19 @@ test('run a live no-op job with one operation', async (t) => { test('run a stringified no-op job with one operation', async (t) => { const job = 'export default [(s) => s]'; const state = createState(); - const result = await executeExpression(job, state); + const context = createContext(); + + const result = await execute(context, job, state); t.deepEqual(state, result); }); test('run a live no-op job with @openfn/language-common.fn', async (t) => { - // @ts-ignore const job = [fn((s) => s)]; const state = createState(); - const result = await executeExpression(job, state); + const context = createContext(); + + const result = await execute(context, job, state); t.deepEqual(state, result); }); @@ -66,52 +82,54 @@ test('run a live no-op job with @openfn/language-common.fn', async (t) => { // it's tricky, from the user's point of view it should just be execution time // ok, fine, we're gonna say that start is literally just execution // but we'll add like a job-initialise event, with a duration -test('call the on-intialise callback', async (t) => { - let didCallCallback = false; +// test('call the on-intialise callback', async (t) => { +// let didCallCallback = false; - const job = [(s: State) => s]; - const state = createState(); +// const job = [(s: State) => s]; +// const state = createState(); - const callbacks = { - onInitStart: () => { - didCallCallback = true; - }, - }; +// const callbacks = { +// onInitStart: () => { +// didCallCallback = true; +// }, +// }; - await executeExpression(job, state, { callbacks }); - t.true(didCallCallback); -}); +// await execute(context, job, state, { callbacks }); +// t.true(didCallCallback); +// }); -test('call the on-intialise-complete callback', async (t) => { - let didCallCallback = false; +// test('call the on-intialise-complete callback', async (t) => { +// let didCallCallback = false; - const job = [(s: State) => s]; - const state = createState(); +// const job = [(s: State) => s]; +// const state = createState(); - const callbacks = { - onInitComplete: ({ duration }: any) => { - t.assert(!isNaN(duration)); - didCallCallback = true; - }, - }; +// const callbacks = { +// onInitComplete: ({ duration }: any) => { +// t.assert(!isNaN(duration)); +// didCallCallback = true; +// }, +// }; - await executeExpression(job, state, { callbacks }); - t.true(didCallCallback); -}); +// await execute(context, job, state, { callbacks }); +// t.true(didCallCallback); +// }); -test('call the on-start callback', async (t) => { +test('notify job-start', async (t) => { let didCallCallback = false; const job = [(s: State) => s]; const state = createState(); - const callbacks = { - onStart: () => { + const notify = (event: string, _payload?: any) => { + if (event === 'job-start') { didCallCallback = true; - }, + } }; - await executeExpression(job, state, { callbacks }); + const context = createContext({ notify }); + + await execute(context, job, state); t.true(didCallCallback); }); @@ -121,15 +139,18 @@ test('call the on-complete callback', async (t) => { const job = [(s: State) => s]; const state = createState(); - const callbacks = { - onComplete: ({ duration, state }: any) => { - t.assert(!isNaN(duration)); - t.truthy(state); + const notify = (event: string, payload: any) => { + if (event === 'job-complete') { + const { state, duration } = payload; didCallCallback = true; - }, + t.truthy(state); + t.assert(!isNaN(duration)); + } }; - await executeExpression(job, state, { callbacks }); + const context = createContext({ notify }); + + await execute(context, job, state); t.true(didCallCallback); }); @@ -139,7 +160,9 @@ test.todo('resolve starting state'); test('jobs can handle a promise', async (t) => { const job = [async (s: State) => s]; const state = createState(); - const result = await executeExpression(job, state); + const context = createContext(); + + const result = await execute(context, job, state); t.deepEqual(state, result); }); @@ -154,7 +177,10 @@ test('output state should be serializable', async (t) => { circular, fn: () => {}, }); - const result = await executeExpression(job, state); + + const context = createContext(); + + const result = await execute(context, job, state); t.notThrows(() => JSON.stringify(result)); @@ -163,24 +189,17 @@ test('output state should be serializable', async (t) => { }); test('config is removed from the result (strict)', async (t) => { - const job = [(s) => s]; + const job = [async (s: State) => s]; + const context = createContext({ opts: { strict: true } }); - const result = await executeExpression( - job, - { configuration: {} }, - { strict: true } - ); + const result = await execute(context, job, { configuration: {} }); t.deepEqual(result, {}); }); test('config is removed from the result (non-strict)', async (t) => { - const job = [(s) => s]; - - const result = await executeExpression( - job, - { configuration: {} }, - { strict: false } - ); + const job = [async (s: State) => s]; + const context = createContext({ opts: { strict: false } }); + const result = await execute(context, job, { configuration: {} }); t.deepEqual(result, {}); }); @@ -194,7 +213,9 @@ test('output state is cleaned in strict mode', async (t) => { }), ]; - const result = await executeExpression(job, {}, { strict: true }); + const context = createContext({ opts: { strict: true } }); + + const result = await execute(context, job, {}); t.deepEqual(result, { data: {}, references: [], @@ -210,7 +231,9 @@ test('output state is left alone in non-strict mode', async (t) => { }; const job = [async () => ({ ...state })]; - const result = await executeExpression(job, {}, { strict: false }); + const context = createContext({ opts: { strict: false } }); + + const result = await execute(context, job, {}); t.deepEqual(result, { data: {}, references: [], @@ -234,11 +257,12 @@ test('operations run in series', async (t) => { }, ] as Operation[]; + const context = createContext(); const state = createState(); // @ts-ignore t.falsy(state.data.x); - const result = (await executeExpression(job, state)) as TestState; + const result = (await execute(context, job, state)) as TestState; t.is(result.data.x, 12); }); @@ -263,10 +287,12 @@ test('async operations run in series', async (t) => { ] as Operation[]; const state = createState(); + const context = createContext(); + // @ts-ignore t.falsy(state.data.x); - const result = (await executeExpression(job, state)) as TestState; + const result = (await execute(context, job, state)) as TestState; t.is(result.data.x, 12); }); @@ -276,7 +302,9 @@ test('jobs can return undefined', async (t) => { const job = [() => undefined] as Operation[]; const state = createState() as TestState; - const result = (await executeExpression(job, state, {})) as TestState; + const context = createContext(); + + const result = (await execute(context, job, state, {})) as TestState; t.assert(result === undefined); }); @@ -290,9 +318,8 @@ test('jobs can mutate the original state', async (t) => { ] as Operation[]; const state = createState({ x: 1 }) as TestState; - const result = (await executeExpression(job, state, { - immutableState: false, - })) as TestState; + const context = createContext({ opts: { immutableState: false } }); + const result = (await execute(context, job, state)) as TestState; t.is(state.data.x, 2); t.is(result.data.x, 2); @@ -307,9 +334,8 @@ test('jobs do not mutate the original state', async (t) => { ] as Operation[]; const state = createState({ x: 1 }) as TestState; - const result = (await executeExpression(job, state, { - immutableState: true, - })) as TestState; + const context = createContext({ opts: { immutableState: true } }); + const result = (await execute(context, job, state)) as TestState; t.is(state.data.x, 1); t.is(result.data.x, 2); @@ -325,7 +351,8 @@ export default [ ];`; const state = createState(); - await executeExpression(job, state, { jobLogger: logger }); + const context = createContext({ opts: { jobLogger: logger } }); + await execute(context, job, state); const output = logger._parse(logger._last); t.is(output.level, 'info'); @@ -341,12 +368,8 @@ test('calls execute if exported from a job', async (t) => { export const execute = () => { console.log('x'); return () => ({}) }; export default []; `; - - await executeExpression( - source, - { configuration: {}, data: {} }, - { jobLogger: logger } - ); + const context = createContext({ opts: { jobLogger: logger } }); + await execute(context, source, { configuration: {}, data: {} }); t.is(logger._history.length, 1); }); @@ -358,12 +381,10 @@ test.skip('Throws after default timeout', async (t) => { const job = `export default [() => new Promise(() => {})];`; const state = createState(); - await t.throwsAsync( - async () => executeExpression(job, state, { jobLogger: logger }), - { - message: 'timeout', - } - ); + const context = createContext({ opts: { jobLogger: logger } }); + await t.throwsAsync(async () => execute(context, job, state), { + message: 'timeout', + }); }); test('Throws after custom timeout', async (t) => { @@ -371,20 +392,20 @@ test('Throws after custom timeout', async (t) => { const job = `export default [() => new Promise((resolve) => setTimeout(resolve, 100))];`; + const context = createContext({ + opts: { jobLogger: logger, timeout: 10 }, + }); const state = createState(); - await t.throwsAsync( - async () => - executeExpression(job, state, { jobLogger: logger, timeout: 10 }), - { - message: 'timeout', - } - ); + await t.throwsAsync(async () => execute(context, job, state), { + message: 'timeout', + }); }); test('Operations log on start and end', async (t) => { const job = [(s: State) => s]; const state = createState(); - await executeExpression(job, state); + const context = createContext(); + await execute(context, job, state); const start = logger._find('debug', /starting operation /i); t.truthy(start); diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index e5793d13f..447b290af 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -23,26 +23,18 @@ test('run a simple workflow', async (t) => { t.true(result.data.done); }); -test('run a workflow and call callbacks', async (t) => { +test('run a workflow and notify major events', async (t) => { const counts: Record = {}; - const increment = (name: string) => { - return { - [name]: () => { - if (!counts[name]) { - counts[name] = 0; - } - counts[name] += 1; - }, - }; + const notify = (name: string) => { + if (!counts[name]) { + counts[name] = 0; + } + counts[name] += 1; }; - const callbacks = Object.assign( - {}, - increment('onInitStart'), - increment('onInitComplete'), - increment('onStart'), - increment('onComplete') - ); + const callbacks = { + notify, + }; const plan: ExecutionPlan = { jobs: [{ expression: 'export default [(s) => s]' }], @@ -50,32 +42,24 @@ test('run a workflow and call callbacks', async (t) => { await run(plan, {}, { callbacks }); - t.is(counts.onInitStart, 1); - t.is(counts.onInitComplete, 1); - t.is(counts.onStart, 1); - t.is(counts.onComplete, 1); + // t.is(counts.onInitStart, 1); + // t.is(counts.onInitComplete, 1); + t.is(counts['job-start'], 1); + t.is(counts['job-complete'], 1); }); test('run a workflow with two jobs and call callbacks', async (t) => { const counts: Record = {}; - const increment = (name: string) => { - return { - [name]: () => { - if (!counts[name]) { - counts[name] = 0; - } - counts[name] += 1; - }, - }; + const notify = (name: string) => { + if (!counts[name]) { + counts[name] = 0; + } + counts[name] += 1; }; - const callbacks = Object.assign( - {}, - increment('onInitStart'), - increment('onInitComplete'), - increment('onStart'), - increment('onComplete') - ); + const callbacks = { + notify, + }; const plan: ExecutionPlan = { jobs: [ @@ -86,10 +70,10 @@ test('run a workflow with two jobs and call callbacks', async (t) => { await run(plan, {}, { callbacks }); - t.is(counts.onInitStart, 2); - t.is(counts.onInitComplete, 2); - t.is(counts.onStart, 2); - t.is(counts.onComplete, 2); + // t.is(counts.onInitStart, 1); + // t.is(counts.onInitComplete, 1); + t.is(counts['job-start'], 2); + t.is(counts['job-complete'], 2); }); test('run a workflow with state and parallel branching', async (t) => { From d4fbcfaf17af9cb867eb2385a63f2ead6f8759c6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 13 Oct 2023 18:34:12 +0100 Subject: [PATCH 146/232] runtime: add handlers for lazy loading credentials and state --- packages/runtime/src/execute/expression.ts | 25 +------ packages/runtime/src/execute/job.ts | 69 ++++++++++++++++--- packages/runtime/src/execute/plan.ts | 13 ++-- packages/runtime/src/runtime.ts | 4 ++ packages/runtime/src/types.ts | 15 ++-- .../runtime/test/execute/expression.test.ts | 52 -------------- packages/runtime/test/execute/plan.test.ts | 38 ++++++++++ packages/runtime/test/runtime.test.ts | 54 +++++++++++++-- 8 files changed, 168 insertions(+), 102 deletions(-) diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts index 9633bd05c..788dbcf2e 100644 --- a/packages/runtime/src/execute/expression.ts +++ b/packages/runtime/src/execute/expression.ts @@ -1,13 +1,7 @@ import { printDuration, Logger } from '@openfn/logger'; import stringify from 'fast-safe-stringify'; import loadModule from '../modules/module-loader'; -import { - Operation, - JobModule, - State, - ExecutionCallbacks, - ExecutionContext, -} from '../types'; +import { Operation, JobModule, State, ExecutionContext } from '../types'; import { Options, ERR_TIMEOUT, TIMEOUT } from '../runtime'; import buildContext, { Context } from './context'; import defaultExecute from '../util/execute'; @@ -25,10 +19,6 @@ export default ( logger.debug('Intialising pipeline'); logger.debug(`Timeout set to ${timeout}ms`); - // let initDuration = Date.now(); - - // callbacks.onInitStart?.(); - // Setup an execution context const context = buildContext(initialState, opts); @@ -39,11 +29,6 @@ export default ( wrapOperation(op, logger, `${idx + 1}`, opts.immutableState) ) ); - // initDuration = Date.now() - initDuration; - - // TODO actually we're going to do the init stuff in job.ts - // But I will notify() for how long it takes to load the job module - // callbacks.onInitComplete?.({ duration: initDuration }); // Run the pipeline try { @@ -98,11 +83,6 @@ const prepareJob = async ( context: Context, opts: Options = {} ): Promise => { - // TODO resolve credential - // TODO resolve initial state - - // difficulty here that credential isn't passed in - if (typeof expression === 'string') { const exports = await loadModule(expression, { ...opts.linker, @@ -141,8 +121,7 @@ const prepareFinalState = (opts: Options, state: any) => { if (state) { if (opts.strict) { state = assignKeys(state, {}, ['data', 'error', 'references']); - } else { - // TODO this is new and needs unit tests + } else if (opts.deleteConfiguration !== false) { delete state.configuration; } const cleanState = stringify(state); diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts index 869a6ba9e..074910912 100644 --- a/packages/runtime/src/execute/job.ts +++ b/packages/runtime/src/execute/job.ts @@ -1,6 +1,9 @@ // TODO hmm. I have a horrible feeling that the callbacks should go here // at least the resolvesrs import executeExpression from './expression'; + +import clone from '../util/clone'; +import assembleState from '../util/assemble-state'; import type { CompiledJobNode, ExecutionContext, @@ -8,32 +11,82 @@ import type { State, } from '../types'; +const loadCredentials = async ( + job: CompiledJobNode, + resolver: (id: string) => Promise +) => { + if (typeof job.configuration === 'string') { + // TODO let's log something useful if we're lazy loading + // TODO throw a controlled error if there's no reoslved + return resolver(job.configuration); + } + return job.configuration; +}; + +const loadState = async ( + job: CompiledJobNode, + resolver: (id: string) => Promise +) => { + if (typeof job.state === 'string') { + // TODO let's log something useful if we're lazy loading + // TODO throw a controlled error if there's no resolver + return resolver(job.state); + } + return job.state; +}; + // The job handler is responsible for preparing the job // and working out where to go next // it'll resolve credentials and state and notify how long init took const executeJob = async ( ctx: ExecutionContext, job: CompiledJobNode, - state: State + initialState: State ): Promise<{ next: JobNodeID[]; state: any }> => { const next: string[] = []; + const { opts, notify, logger, report } = ctx; + + const duration = Date.now(); + + notify?.('init-start'); + + // lazy load config and state + const configuration = await loadCredentials( + job, + opts.callbacks?.resolveCredential! // cheat - we need to handle the error case here + ); + + const globals = await loadState( + job, + opts.callbacks?.resolveState! // and here + ); + + const state = assembleState( + clone(initialState), + configuration, + globals, + opts.strict + ); + + notify?.('init-complete', { duration: Date.now() - duration }); + // We should by this point have validated the plan, so the job MUST exist - ctx.logger.timer('job'); - ctx.logger.always('Starting job', job.id); + logger.timer('job'); + logger.always('Starting job', job.id); let result: any = state; if (job.expression) { // The expression SHOULD return state, but could return anything try { result = await executeExpression(ctx, job.expression, state); - const duration = ctx.logger.timer('job'); - ctx.logger.success(`Completed job ${job.id} in ${duration}`); + const duration = logger.timer('job'); + logger.success(`Completed job ${job.id} in ${duration}`); } catch (e: any) { - const duration = ctx.logger.timer('job'); - ctx.logger.error(`Failed job ${job.id} after ${duration}`); - ctx.report(state, job.id, e); + const duration = logger.timer('job'); + logger.error(`Failed job ${job.id} after ${duration}`); + report(state, job.id, e); } } diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index fe85fa370..c52147e4e 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -1,10 +1,9 @@ import type { Logger } from '@openfn/logger'; import executeJob from './job'; import compilePlan from './compile-plan'; -import assembleState from '../util/assemble-state'; + import type { ExecutionPlan, State } from '../types'; import type { Options } from '../runtime'; -import clone from '../util/clone'; import validatePlan from '../util/validate-plan'; import createErrorReporter from '../util/log-error'; @@ -39,20 +38,16 @@ const executePlan = async ( // Record of state on lead nodes (nodes with no next) const leaves: Record = {}; + // TODO: maybe lazy load intial state and notify about it + // Right now this executes in series, even if jobs are parallelised while (queue.length) { const next = queue.shift()!; const job = compiledPlan.jobs[next]; const prevState = stateHistory[job.previous || ''] ?? initialState; - const state = assembleState( - clone(prevState), - job.configuration, - job.state, - ctx.opts.strict - ); - const result = await executeJob(ctx, job, state); + const result = await executeJob(ctx, job, prevState); stateHistory[next] = result.state; if (!result.next.length) { diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 47e18085b..0f262eeaa 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -23,6 +23,7 @@ export type Options = { timeout?: number; strict?: boolean; // Be strict about handling of state returned from jobs + deleteConfiguration?: boolean; // Treat state as immutable (likely to break in legacy jobs) immutableState?: boolean; @@ -55,6 +56,9 @@ const run = ( if (!opts.hasOwnProperty('strict')) { opts.strict = true; } + if (!opts.hasOwnProperty('deleteConfiguration')) { + opts.deleteConfiguration = true; + } // TODO the plan doesn't have an id, should it be given one? // Ditto the jobs? diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index a4df7578f..0028f6249 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -61,8 +61,8 @@ export type JobNode = { expression?: string | Operation[]; // the code we actually want to execute. Can be a path. - configuration?: object; // credential object - state?: Omit; // default state (globals) + configuration?: object | string; // credential object + state?: Omit | string; // default state (globals) next?: string | Record; previous?: JobNodeID; @@ -110,17 +110,20 @@ export type ExecutionContext = { notify: (evt: NotifyEvents, payload?: any) => void; }; +// External notifications to reveal what's happening +// All are passed through the notify handler export type NotifyEvents = - | 'init-start' + | 'init-start' // The job is being initialised (ie, credentials lazy-loading) | 'init-complete' | 'job-start' | 'job-complete'; +// module-load // I'm not wild about the callbacks pattern, events would be simpler // but a) the resolvers MUST be a callback and b) we don't currently have an event emitter API // no, ok, we're going to have a notify function which does the callbacks export type ExecutionCallbacks = { - notify(event: NotifyEvents, payload: any): void; - resolveState?: (state: string) => any; - resolveCredential?: (state: string) => any; + notify?(event: NotifyEvents, payload: any): void; + resolveState?: (stateId: string) => Promise; + resolveCredential?: (credentialId: string) => Promise; }; diff --git a/packages/runtime/test/execute/expression.test.ts b/packages/runtime/test/execute/expression.test.ts index 56f211688..8cdab5588 100644 --- a/packages/runtime/test/execute/expression.test.ts +++ b/packages/runtime/test/execute/expression.test.ts @@ -27,12 +27,6 @@ const createContext = (args = {}) => ...args, } as unknown as ExecutionContext); -// const executeExpression = ( -// job: string | Operation[], -// state: State, -// opts = {} -// ) => execute(job, state, logger, opts); - test.afterEach(() => { logger._reset(); }); @@ -72,49 +66,6 @@ test('run a live no-op job with @openfn/language-common.fn', async (t) => { t.deepEqual(state, result); }); -// TODO all these need implementing -// do they happen in expression.ts or elsewhere? -// I think this is fine - -// on-start is called before things are resolved -// it means "i'm about to start processing this job" -// does duration include state loading? -// it's tricky, from the user's point of view it should just be execution time -// ok, fine, we're gonna say that start is literally just execution -// but we'll add like a job-initialise event, with a duration -// test('call the on-intialise callback', async (t) => { -// let didCallCallback = false; - -// const job = [(s: State) => s]; -// const state = createState(); - -// const callbacks = { -// onInitStart: () => { -// didCallCallback = true; -// }, -// }; - -// await execute(context, job, state, { callbacks }); -// t.true(didCallCallback); -// }); - -// test('call the on-intialise-complete callback', async (t) => { -// let didCallCallback = false; - -// const job = [(s: State) => s]; -// const state = createState(); - -// const callbacks = { -// onInitComplete: ({ duration }: any) => { -// t.assert(!isNaN(duration)); -// didCallCallback = true; -// }, -// }; - -// await execute(context, job, state, { callbacks }); -// t.true(didCallCallback); -// }); - test('notify job-start', async (t) => { let didCallCallback = false; @@ -154,9 +105,6 @@ test('call the on-complete callback', async (t) => { t.true(didCallCallback); }); -test.todo('resolve a credential'); -test.todo('resolve starting state'); - test('jobs can handle a promise', async (t) => { const job = [async (s: State) => s]; const state = createState(); diff --git a/packages/runtime/test/execute/plan.test.ts b/packages/runtime/test/execute/plan.test.ts index 12b8a7c44..11885eeed 100644 --- a/packages/runtime/test/execute/plan.test.ts +++ b/packages/runtime/test/execute/plan.test.ts @@ -139,6 +139,37 @@ test('execute a one-job execution plan with initial state', async (t) => { t.is(result, 33); }); +test('execute a one-job execution plan and notify init-start and init-complete', async (t) => { + let notifications: Record = {}; + + const plan: ExecutionPlan = { + jobs: [ + { + expression: 'export default [s => s.data.x]', + }, + ], + }; + + const notify = (event: string, payload: any) => { + if (notifications[event]) { + throw new Error(`event ${event} called twice!`); + } + notifications[event] = payload || true; + }; + + const options = { callbacks: { notify } }; + + const state = { + data: { x: 33 }, + }; + + await executePlan(plan, state, options); + + t.truthy(notifications['init-start']); + t.truthy(notifications['init-complete']); + t.assert(!isNaN(notifications['init-complete'].duration)); +}); + test('execute a job with a simple truthy "precondition" or "trigger node"', async (t) => { const plan: ExecutionPlan = { jobs: [ @@ -928,3 +959,10 @@ test('Plans log for each job start and end', async (t) => { const end = logger._find('success', /completed job/i); t.regex(end!.message as string, /Completed job a in \d+ms/); }); + +// TODO these are part of job.ts really, should we split tests up into plan and job? +// There's quite a big overlap really in test terms +test.todo('lazy load credentials'); +test.todo("throw if credentials are a string and there's no resolver"); +test.todo('lazy load state'); +test.todo("throw if state is a string and there's no resolver"); diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts index 447b290af..8f1542d87 100644 --- a/packages/runtime/test/runtime.test.ts +++ b/packages/runtime/test/runtime.test.ts @@ -42,12 +42,58 @@ test('run a workflow and notify major events', async (t) => { await run(plan, {}, { callbacks }); - // t.is(counts.onInitStart, 1); - // t.is(counts.onInitComplete, 1); + t.is(counts['init-start'], 1); + t.is(counts['init-complete'], 1); t.is(counts['job-start'], 1); t.is(counts['job-complete'], 1); }); +test('resolve a credential', async (t) => { + const plan: ExecutionPlan = { + jobs: [ + { + expression: 'export default [(s) => s]', + configuration: 'ccc', + }, + ], + }; + + const options = { + strict: false, + deleteConfiguration: false, + callbacks: { + resolveCredential: async () => ({ password: 'password1' }), + }, + }; + + const result: any = await run(plan, {}, options); + t.truthy(result); + t.deepEqual(result.configuration, { password: 'password1' }); +}); + +test.todo('resolve intial state'); + +test('resolve initial state', async (t) => { + const plan: ExecutionPlan = { + jobs: [ + { + expression: 'export default [(s) => s]', + state: 'abc', + }, + ], + }; + + const options = { + callbacks: { + resolveState: async () => ({ data: { foo: 'bar' } }), + }, + }; + + const result: any = await run(plan, {}, options); + t.truthy(result); + t.deepEqual(result.data, { foo: 'bar' }); +}); + test('run a workflow with two jobs and call callbacks', async (t) => { const counts: Record = {}; const notify = (name: string) => { @@ -70,8 +116,8 @@ test('run a workflow with two jobs and call callbacks', async (t) => { await run(plan, {}, { callbacks }); - // t.is(counts.onInitStart, 1); - // t.is(counts.onInitComplete, 1); + t.is(counts['init-start'], 2); + t.is(counts['init-complete'], 2); t.is(counts['job-start'], 2); t.is(counts['job-complete'], 2); }); From d3580cbd9d099c10b899cdfec3ff8b46223a5631 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 10:49:26 +0100 Subject: [PATCH 147/232] engine: lazy load intial state --- packages/runtime/src/execute/compile-plan.ts | 1 + packages/runtime/src/execute/job.ts | 8 +- packages/runtime/src/execute/plan.ts | 18 +- packages/runtime/src/runtime.ts | 5 +- packages/runtime/src/types.ts | 9 +- packages/runtime/src/util/notify.ts | 7 +- packages/runtime/test/errors.test.ts | 27 +++ .../runtime/test/execute/compile-plan.test.ts | 19 ++ packages/runtime/test/execute/plan.test.ts | 168 ++++++++++-------- 9 files changed, 175 insertions(+), 87 deletions(-) create mode 100644 packages/runtime/test/errors.test.ts diff --git a/packages/runtime/src/execute/compile-plan.ts b/packages/runtime/src/execute/compile-plan.ts index 1a0487b6d..f5c7291c0 100644 --- a/packages/runtime/src/execute/compile-plan.ts +++ b/packages/runtime/src/execute/compile-plan.ts @@ -99,6 +99,7 @@ export default (plan: ExecutionPlan) => { const newPlan = { jobs: {}, start: plan.start, + initialState: plan.initialState, } as Pick; for (const job of plan.jobs) { diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts index 074910912..0471ff109 100644 --- a/packages/runtime/src/execute/job.ts +++ b/packages/runtime/src/execute/job.ts @@ -16,8 +16,8 @@ const loadCredentials = async ( resolver: (id: string) => Promise ) => { if (typeof job.configuration === 'string') { - // TODO let's log something useful if we're lazy loading - // TODO throw a controlled error if there's no reoslved + // TODO let's log and notify something useful if we're lazy loading + // TODO throw a controlled error if there's no reoslver return resolver(job.configuration); } return job.configuration; @@ -28,7 +28,7 @@ const loadState = async ( resolver: (id: string) => Promise ) => { if (typeof job.state === 'string') { - // TODO let's log something useful if we're lazy loading + // TODO let's log and notify something useful if we're lazy loading // TODO throw a controlled error if there's no resolver return resolver(job.state); } @@ -41,7 +41,7 @@ const loadState = async ( const executeJob = async ( ctx: ExecutionContext, job: CompiledJobNode, - initialState: State + initialState: State = {} ): Promise<{ next: JobNodeID[]; state: any }> => { const next: string[] = []; diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index c52147e4e..f9ad3093a 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -6,10 +6,10 @@ import type { ExecutionPlan, State } from '../types'; import type { Options } from '../runtime'; import validatePlan from '../util/validate-plan'; import createErrorReporter from '../util/log-error'; +import { NOTIFY_STATE_LOAD } from '../util/notify'; const executePlan = async ( plan: ExecutionPlan, - initialState: State = {}, opts: Options, logger: Logger ) => { @@ -38,7 +38,21 @@ const executePlan = async ( // Record of state on lead nodes (nodes with no next) const leaves: Record = {}; - // TODO: maybe lazy load intial state and notify about it + const { initialState } = compiledPlan; + + if (typeof initialState === 'string') { + const id = initialState; + const startTime = Date.now(); + logger.debug(`fetching intial state ${id}`); + + initialState = await opts.callbacks?.resolveState(id); + + const duration = Date.now() - startTime; + opts.callbacks?.notify?.(NOTIFY_STATE_LOAD, { duration, id }); + logger.success(`loaded state for ${id} in ${duration}ms`); + + // TODO catch and re-throw + } // Right now this executes in series, even if jobs are parallelised while (queue.length) { diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 0f262eeaa..3ecaf7be2 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -81,8 +81,11 @@ const run = ( } else { plan = expressionOrXPlan as ExecutionPlan; } + if (state) { + plan.initialState = clone(state); + } - return executePlan(plan, clone(state), opts, logger); + return executePlan(plan, opts, logger); }; export default run; diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 0028f6249..41cba641c 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -49,7 +49,7 @@ export type ExecutionPlan = { id?: string; // UUID for this plan jobs: JobNode[]; start?: JobNodeID; - initialState?: State; // TODO adding initial state to the plan changes how the runtime expects to receive initial state + initialState?: State | string; }; export type JobNode = { @@ -62,6 +62,8 @@ export type JobNode = { expression?: string | Operation[]; // the code we actually want to execute. Can be a path. configuration?: object | string; // credential object + + // TODO strings aren't actually suppored here yet state?: Omit | string; // default state (globals) next?: string | Record; @@ -93,6 +95,7 @@ export type CompiledExecutionPlan = { id?: string; start: JobNodeID; jobs: Record; + initialState?: State | string; }; export type JobModule = { @@ -112,11 +115,13 @@ export type ExecutionContext = { // External notifications to reveal what's happening // All are passed through the notify handler +// TODO I'd prefer to load the strings from notify.ts and reflect the types here export type NotifyEvents = | 'init-start' // The job is being initialised (ie, credentials lazy-loading) | 'init-complete' | 'job-start' - | 'job-complete'; + | 'job-complete' + | 'load-state'; // module-load // I'm not wild about the callbacks pattern, events would be simpler diff --git a/packages/runtime/src/util/notify.ts b/packages/runtime/src/util/notify.ts index cc4043edd..0cbb52655 100644 --- a/packages/runtime/src/util/notify.ts +++ b/packages/runtime/src/util/notify.ts @@ -1,5 +1,2 @@ -function createNotifier(callback) { - return (evt, payload) => { - callback(evt, payload); - }; -} +// TODO add all notify strings here +export const NOTIFY_STATE_LOAD = 'load-state'; diff --git a/packages/runtime/test/errors.test.ts b/packages/runtime/test/errors.test.ts new file mode 100644 index 000000000..2548aab95 --- /dev/null +++ b/packages/runtime/test/errors.test.ts @@ -0,0 +1,27 @@ +/** + reproduce various errors and test how the runtime responds + it should basically throw with a set of expected error cases + + InputError - the workflow structure failed validation + RuntimeError - error while executing. This could have a subtype like TypeError, ReferenceError + It is a bit of a confusing name, is JobError, ExpressionError or ExeuctionError better? + CompileError - error while compiling code, probably a syntax error + LinkerError - basically a problem loading any dependency (probably the adaptor) + ModuleError? DependencyError? ImportError? + TimeoutError - a job ran for too long + ResolveError - a state or credential resolver failed (is this an input error?) + + what about code generation errors? That'll be a RuntimeError, should we treat it spedcially? + SecurityError maybe? + + Note that there are errors we can't catch here, like memory or diskspace blowups, infinite loops. + It's the worker's job to catch those and report the crash + + We'll have a RuntimeError type which has as reason string (that gets forwarded to the worker) + a type and subtype, and a message + + Later we'll do stacktraces and positions and stuff but not now. Maybe for a JobError I guess? + */ +import test from 'ava'; + +test.todo('errors'); diff --git a/packages/runtime/test/execute/compile-plan.test.ts b/packages/runtime/test/execute/compile-plan.test.ts index 40de866a6..23ef3518e 100644 --- a/packages/runtime/test/execute/compile-plan.test.ts +++ b/packages/runtime/test/execute/compile-plan.test.ts @@ -17,6 +17,25 @@ const planWithEdge = (edge: JobEdge) => jobs: [{ id: 'a', next: { b: edge } }], } as ExecutionPlan); +test('should preserve initial state as an object', (t) => { + const state = { x: 123 }; + const compiledPlan = compilePlan({ + id: 'a', + initialState: state, + jobs: [], + }); + t.deepEqual(state, compiledPlan.initialState); +}); + +test('should preserve initial state a string', (t) => { + const compiledPlan = compilePlan({ + id: 'a', + initialState: 'abc', + jobs: [], + }); + t.is(compiledPlan.initialState, 'abc'); +}); + test('should convert jobs to an object', (t) => { const compiledPlan = compilePlan(testPlan); t.truthy(compiledPlan.jobs.a); diff --git a/packages/runtime/test/execute/plan.test.ts b/packages/runtime/test/execute/plan.test.ts index 11885eeed..2696eb3cd 100644 --- a/packages/runtime/test/execute/plan.test.ts +++ b/packages/runtime/test/execute/plan.test.ts @@ -4,16 +4,8 @@ import { createMockLogger } from '@openfn/logger'; import { ExecutionPlan, JobNode } from '../../src/types'; import execute from './../../src/execute/plan'; -const opts = {}; let mockLogger = createMockLogger(undefined, { level: 'debug' }); -const executePlan = ( - plan: ExecutionPlan, - state = {}, - options = opts, - logger = mockLogger -): any => execute(plan, state, options, logger); - test('throw for a circular job', async (t) => { const plan: ExecutionPlan = { start: 'job1', @@ -30,7 +22,7 @@ test('throw for a circular job', async (t) => { }, ], }; - const e = await t.throwsAsync(() => executePlan(plan)); + const e = await t.throwsAsync(() => execute(plan, {}, mockLogger)); t.regex(e!.message, /circular dependency/i); }); @@ -56,7 +48,7 @@ test('throw for a job with multiple inputs', async (t) => { }, ], }; - const e = await t.throwsAsync(() => executePlan(plan)); + const e = await t.throwsAsync(() => execute(plan, {}, mockLogger)); t.regex(e!.message, /multiple dependencies/i); }); @@ -71,7 +63,7 @@ test('throw for a plan which references an undefined job', async (t) => { }, ], }; - const e = await t.throwsAsync(() => executePlan(plan)); + const e = await t.throwsAsync(() => execute(plan, {}, mockLogger)); t.regex(e!.message, /cannot find job/i); }); @@ -90,7 +82,7 @@ test('throw for an illegal edge condition', async (t) => { { id: 'b' }, ], }; - const e = await t.throwsAsync(() => executePlan(plan)); + const e = await t.throwsAsync(() => execute(plan, {}, mockLogger)); t.regex(e!.message, /failed to compile edge condition a->b/i); }); @@ -108,7 +100,7 @@ test('throw for an edge condition', async (t) => { { id: 'b' }, ], }; - const e = await t.throwsAsync(() => executePlan(plan)); + const e = await t.throwsAsync(() => execute(plan, {}, mockLogger)); t.regex(e!.message, /failed to compile edge condition/i); }); @@ -121,28 +113,53 @@ test('execute a one-job execution plan with inline state', async (t) => { }, ], }; - const result = (await executePlan(plan)) as unknown as number; + const result = (await execute(plan, {}, mockLogger)) as unknown as number; t.is(result, 22); }); test('execute a one-job execution plan with initial state', async (t) => { const plan: ExecutionPlan = { + initialState: { + data: { x: 33 }, + }, jobs: [ { expression: 'export default [s => s.data.x]', }, ], }; - const result = (await executePlan(plan, { - data: { x: 33 }, - })) as unknown as number; + const result = (await execute(plan, {}, mockLogger)) as unknown as number; t.is(result, 33); }); +test('lazy load initial state', async (t) => { + const plan: ExecutionPlan = { + initialState: 's1', + jobs: [{ id: 'a', expression: 'export default [s => s]' }], + }; + const states = { s1: { data: { result: 42 } } }; + const options = { + callbacks: { + resolveState: (id: string) => states[id], + }, + }; + + const result = await execute(plan, options, mockLogger); + t.deepEqual(result, states.s1); +}); + +test.todo('lazy load initial state with log'); +test.todo('lazy load initial state with notify'); + test('execute a one-job execution plan and notify init-start and init-complete', async (t) => { let notifications: Record = {}; + const state = { + data: { x: 33 }, + }; + const plan: ExecutionPlan = { + initialState: state, jobs: [ { expression: 'export default [s => s.data.x]', @@ -159,11 +176,7 @@ test('execute a one-job execution plan and notify init-start and init-complete', const options = { callbacks: { notify } }; - const state = { - data: { x: 33 }, - }; - - await executePlan(plan, state, options); + await execute(plan, options, mockLogger); t.truthy(notifications['init-start']); t.truthy(notifications['init-complete']); @@ -186,7 +199,7 @@ test('execute a job with a simple truthy "precondition" or "trigger node"', asyn }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.true(result.data.done); }); @@ -206,12 +219,13 @@ test('do not execute a job with a simple falsy "precondition" or "trigger node"' }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.falsy(result.data.done); }); test('execute a job with a valid "precondition" or "trigger node"', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 10 } }, jobs: [ { next: { @@ -226,12 +240,13 @@ test('execute a job with a valid "precondition" or "trigger node"', async (t) => }, ], }; - const result = await executePlan(plan, { data: { x: 10 } }); + const result = await execute(plan, {}, mockLogger); t.true(result.data.done); }); test('merge initial and inline state', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 33 } }, jobs: [ { expression: 'export default [s => s]', @@ -239,13 +254,14 @@ test('merge initial and inline state', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 33 } }); + const result = await execute(plan, {}, mockLogger); t.is(result.data.x, 33); t.is(result.data.y, 11); }); test('Initial state overrides inline data', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 34 } }, jobs: [ { expression: 'export default [s => s]', @@ -253,8 +269,8 @@ test('Initial state overrides inline data', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 33 } }); - t.is(result.data.x, 33); + const result = await execute(plan, {}, mockLogger); + t.is(result.data.x, 34); }); test('Previous state overrides inline data', async (t) => { @@ -278,7 +294,7 @@ test('Previous state overrides inline data', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.data.x, 6); }); @@ -301,7 +317,7 @@ test('only allowed state is passed through in strict mode', async (t) => { }, ], }; - const result = await executePlan(plan, {}, { strict: true }); + const result = await execute(plan, { strict: true }, mockLogger); t.deepEqual(result, { data: {}, references: [], @@ -364,7 +380,7 @@ test('Jobs only receive state from upstream jobs', async (t) => { ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); // explicit check that no assertion failed and wrote an error to state t.falsy(result.error); @@ -395,7 +411,7 @@ test('all state is passed through in non-strict mode', async (t) => { }, ], }; - const result = await executePlan(plan, {}, { strict: false }); + const result = await execute(plan, { strict: false }, mockLogger); t.deepEqual(result, { data: {}, references: [], @@ -421,7 +437,7 @@ test('execute edge based on state in the condition', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.data?.y, 20); }); @@ -442,12 +458,13 @@ test('skip edge based on state in the condition ', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.data?.x, 10); }); test('execute a two-job execution plan', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 0 } }, jobs: [ { id: 'job1', @@ -460,12 +477,13 @@ test('execute a two-job execution plan', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 0 } }); + const result = await execute(plan, {}, mockLogger); t.is(result.data.x, 2); }); test('only execute one job in a two-job execution plan', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 0 } }, jobs: [ { id: 'job1', @@ -478,7 +496,7 @@ test('only execute one job in a two-job execution plan', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 0 } }); + const result = await execute(plan, {}, mockLogger); t.is(result.data.x, 1); }); @@ -497,13 +515,14 @@ test('execute a two-job execution plan with custom start in state', async (t) => }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.data.result, 11); }); test('execute a two-job execution plan with custom start in options', async (t) => { const plan: ExecutionPlan = { start: 'job1', + initialState: { start: 'job2' }, jobs: [ { id: 'job1', @@ -516,13 +535,14 @@ test('execute a two-job execution plan with custom start in options', async (t) }, ], }; - const result = await executePlan(plan, {}, { start: 'job2' }); + const result = await execute(plan, {}, mockLogger); t.is(result.data.result, 11); }); test('Return when there are no more edges', async (t) => { const plan: ExecutionPlan = { start: 'job1', + initialState: { data: { x: 0 } }, jobs: [ { id: 'job1', @@ -534,12 +554,13 @@ test('Return when there are no more edges', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 0 } }); + const result = await execute(plan, {}, mockLogger); t.is(result.data?.x, 1); }); test('execute a 5 job execution plan', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 0 } }, start: '1', jobs: [], } as ExecutionPlan; @@ -550,13 +571,14 @@ test('execute a 5 job execution plan', async (t) => { next: i === 5 ? null : { [`${i + 1}`]: true }, } as JobNode); } - const result = await executePlan(plan, { data: { x: 0 } }); + const result = await execute(plan, {}, mockLogger); t.is(result.data.x, 5); }); test('execute multiple steps in "parallel"', async (t) => { const plan: ExecutionPlan = { start: 'start', + initialState: { data: { x: 0 } }, jobs: [ { id: 'start', @@ -581,7 +603,7 @@ test('execute multiple steps in "parallel"', async (t) => { }, ], }; - const result = await executePlan(plan, { data: { x: 0 } }); + const result = await execute(plan, {}, mockLogger); t.deepEqual(result, { a: { data: { x: 1 } }, b: { data: { x: 1 } }, @@ -599,7 +621,7 @@ test('return an error in state', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.truthy(result.errors); t.is(result.errors.a.message, 'e'); }); @@ -615,7 +637,7 @@ test('handle non-standard error objects', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.truthy(result.errors); t.is(result.errors.a.error, 'wibble'); }); @@ -637,7 +659,7 @@ test('keep executing after an error', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.y, 20); t.falsy(result.x); }); @@ -664,7 +686,7 @@ test('simple on-error handler', async (t) => { }, ], }; - const result = await executePlan(plan); + const result = await execute(plan, {}, mockLogger); t.is(result.y, 20); t.falsy(result.x); }); @@ -682,7 +704,7 @@ test('log appopriately on error', async (t) => { const logger = createMockLogger(undefined, { level: 'debug' }); - await executePlan(plan, {}, {}, logger); + await execute(plan, {}, logger); const err = logger._find('error', /failed job/i); t.truthy(err); @@ -694,6 +716,7 @@ test('log appopriately on error', async (t) => { test('jobs do not share a local scope', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { // declare x in this expression's scope @@ -709,7 +732,7 @@ test('jobs do not share a local scope', async (t) => { }, ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); const err = result.errors['b']; t.truthy(err); @@ -719,6 +742,7 @@ test('jobs do not share a local scope', async (t) => { test('jobs do not share a global scope', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { expression: 'export default [s => { x = 10; return s; }]', @@ -732,7 +756,7 @@ test('jobs do not share a global scope', async (t) => { }, ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); const err = result.errors['b']; t.truthy(err); @@ -742,6 +766,7 @@ test('jobs do not share a global scope', async (t) => { test('jobs do not share a this object', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { expression: 'export default [s => { this.x = 10; return s; }]', @@ -755,7 +780,7 @@ test('jobs do not share a this object', async (t) => { }, ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); const err = result.errors['b']; t.truthy(err); @@ -767,6 +792,7 @@ test('jobs do not share a this object', async (t) => { // https://github.com/OpenFn/kit/issues/213 test.skip('jobs cannot scribble on globals', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { expression: 'export default [s => { console.x = 10; return s; }]', @@ -780,7 +806,7 @@ test.skip('jobs cannot scribble on globals', async (t) => { }, ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); t.falsy(result.data.x); }); @@ -788,6 +814,7 @@ test.skip('jobs cannot scribble on globals', async (t) => { // https://github.com/OpenFn/kit/issues/213 test.skip('jobs cannot scribble on adaptor functions', async (t) => { const plan: ExecutionPlan = { + initialState: { data: { x: 0 } }, jobs: [ { expression: @@ -803,19 +830,17 @@ test.skip('jobs cannot scribble on adaptor functions', async (t) => { }, ], }; - const result = await executePlan( - plan, - { data: { x: 0 } }, - { - linker: { - modules: { - '@openfn/language-common': { - path: path.resolve('test/__modules__/@openfn/language-common'), - }, + const options = { + linker: { + modules: { + '@openfn/language-common': { + path: path.resolve('test/__modules__/@openfn/language-common'), }, }, - } - ); + }, + }; + + const result = await execute(plan, options, mockLogger); t.falsy(result.data.x); }); @@ -830,6 +855,7 @@ test('jobs can write circular references to state without blowing up downstream' }] `; const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { expression, @@ -842,7 +868,7 @@ test('jobs can write circular references to state without blowing up downstream' ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); t.notThrows(() => JSON.stringify(result)); t.deepEqual(result, { @@ -865,6 +891,7 @@ test('jobs cannot pass circular references to each other', async (t) => { }] `; const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { expression, @@ -880,7 +907,7 @@ test('jobs cannot pass circular references to each other', async (t) => { ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); t.notThrows(() => JSON.stringify(result)); t.is(result.data.answer, '[Circular]'); @@ -888,6 +915,7 @@ test('jobs cannot pass circular references to each other', async (t) => { test('jobs can write functions to state without blowing up downstream', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { next: { b: true }, @@ -906,7 +934,7 @@ test('jobs can write functions to state without blowing up downstream', async (t ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); t.notThrows(() => JSON.stringify(result)); t.deepEqual(result, { data: {} }); @@ -914,6 +942,7 @@ test('jobs can write functions to state without blowing up downstream', async (t test('jobs cannot pass functions to each other', async (t) => { const plan: ExecutionPlan = { + initialState: { data: {} }, jobs: [ { next: { b: true }, @@ -934,7 +963,7 @@ test('jobs cannot pass functions to each other', async (t) => { ], }; - const result = await executePlan(plan, { data: {} }); + const result = await execute(plan, {}, mockLogger); const err = result.errors['b']; t.truthy(err); t.is(err.message, 's.data.x is not a function'); @@ -951,7 +980,7 @@ test('Plans log for each job start and end', async (t) => { ], }; const logger = createMockLogger(undefined, { level: 'debug' }); - await executePlan(plan, {}, {}, logger); + await execute(plan, {}, logger); const start = logger._find('always', /starting job/i); t.is(start!.message, 'Starting job a'); @@ -959,10 +988,3 @@ test('Plans log for each job start and end', async (t) => { const end = logger._find('success', /completed job/i); t.regex(end!.message as string, /Completed job a in \d+ms/); }); - -// TODO these are part of job.ts really, should we split tests up into plan and job? -// There's quite a big overlap really in test terms -test.todo('lazy load credentials'); -test.todo("throw if credentials are a string and there's no resolver"); -test.todo('lazy load state'); -test.todo("throw if state is a string and there's no resolver"); From 65b87e5abf6e7e5fa9bd7a8d1caee8af5ac9175a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 10:52:25 +0100 Subject: [PATCH 148/232] engine: typings --- packages/runtime/src/execute/plan.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index f9ad3093a..1b28f0c9c 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -2,7 +2,7 @@ import type { Logger } from '@openfn/logger'; import executeJob from './job'; import compilePlan from './compile-plan'; -import type { ExecutionPlan, State } from '../types'; +import type { ExecutionPlan } from '../types'; import type { Options } from '../runtime'; import validatePlan from '../util/validate-plan'; import createErrorReporter from '../util/log-error'; @@ -29,7 +29,7 @@ const executePlan = async ( opts, logger, report: createErrorReporter(logger), - notify: opts.callbacks?.notify, + notify: opts.callbacks?.notify ?? (() => {}), }; type State = any; @@ -38,14 +38,14 @@ const executePlan = async ( // Record of state on lead nodes (nodes with no next) const leaves: Record = {}; - const { initialState } = compiledPlan; + let { initialState } = compiledPlan; if (typeof initialState === 'string') { const id = initialState; const startTime = Date.now(); logger.debug(`fetching intial state ${id}`); - initialState = await opts.callbacks?.resolveState(id); + initialState = await opts.callbacks?.resolveState?.(id); const duration = Date.now() - startTime; opts.callbacks?.notify?.(NOTIFY_STATE_LOAD, { duration, id }); From a4fe705ff0f04095395f70a24e89195337d595c0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 11:25:30 +0100 Subject: [PATCH 149/232] mock-lightning: break out the lightning mock into its own server. We need this for integration tests --- packages/lightning-mock/README.md | 10 + packages/lightning-mock/package.json | 53 +++ packages/lightning-mock/src/api-dev.ts | 186 ++++++++++ packages/lightning-mock/src/api-sockets.ts | 335 +++++++++++++++++ packages/lightning-mock/src/events.ts | 68 ++++ packages/lightning-mock/src/index.ts | 2 + packages/lightning-mock/src/server.ts | 99 +++++ packages/lightning-mock/src/socket-server.ts | 268 ++++++++++++++ packages/lightning-mock/src/start.ts | 29 ++ packages/lightning-mock/src/types.ts | 32 ++ packages/lightning-mock/src/util.ts | 15 + packages/lightning-mock/test/data.ts | 32 ++ .../lightning-mock/test/lightning.test.ts | 345 ++++++++++++++++++ packages/lightning-mock/tsconfig.json | 7 + 14 files changed, 1481 insertions(+) create mode 100644 packages/lightning-mock/README.md create mode 100644 packages/lightning-mock/package.json create mode 100644 packages/lightning-mock/src/api-dev.ts create mode 100644 packages/lightning-mock/src/api-sockets.ts create mode 100644 packages/lightning-mock/src/events.ts create mode 100644 packages/lightning-mock/src/index.ts create mode 100644 packages/lightning-mock/src/server.ts create mode 100644 packages/lightning-mock/src/socket-server.ts create mode 100644 packages/lightning-mock/src/start.ts create mode 100644 packages/lightning-mock/src/types.ts create mode 100644 packages/lightning-mock/src/util.ts create mode 100644 packages/lightning-mock/test/data.ts create mode 100644 packages/lightning-mock/test/lightning.test.ts create mode 100644 packages/lightning-mock/tsconfig.json diff --git a/packages/lightning-mock/README.md b/packages/lightning-mock/README.md new file mode 100644 index 000000000..3bf01c1c1 --- /dev/null +++ b/packages/lightning-mock/README.md @@ -0,0 +1,10 @@ +## Lightning Mock + +This package contains a mock lightning server, designed to be used with the worker + +It is mostly used for unit and integration tests, but it can be run standalone from the command line: + +``` +pnpm install +pnpm start +``` diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json new file mode 100644 index 000000000..4a0652c55 --- /dev/null +++ b/packages/lightning-mock/package.json @@ -0,0 +1,53 @@ +{ + "name": "@openfn/lightning-mock", + "version": "1.0.0", + "private": true, + "description": "A mock Lightning server", + "main": "index.js", + "type": "module", + "scripts": { + "test": "pnpm ava --serial", + "test:types": "pnpm tsc --noEmit --project tsconfig.json", + "build": "tsup --config ../../tsup.config.js src/index.ts --no-splitting", + "build:watch": "pnpm build --watch", + "start": "ts-node-esm --transpile-only src/start.ts", + "_pack": "pnpm pack --pack-destination ../../dist" + }, + "author": "Open Function Group ", + "license": "ISC", + "dependencies": { + "@koa/router": "^12.0.0", + "@openfn/engine-multi": "workspace:*", + "@openfn/logger": "workspace:*", + "@openfn/runtime": "workspace:*", + "@types/koa-logger": "^3.1.2", + "@types/ws": "^8.5.6", + "fast-safe-stringify": "^2.1.1", + "jose": "^4.14.6", + "koa": "^2.13.4", + "koa-bodyparser": "^4.4.0", + "koa-logger": "^3.2.1", + "phoenix": "^1.7.7", + "ws": "^8.14.1" + }, + "devDependencies": { + "@types/koa": "^2.13.5", + "@types/koa-bodyparser": "^4.3.10", + "@types/koa-route": "^3.2.6", + "@types/koa-websocket": "^5.0.8", + "@types/koa__router": "^12.0.1", + "@types/node": "^18.15.3", + "@types/nodemon": "1.19.3", + "@types/phoenix": "^1.6.2", + "@types/yargs": "^17.0.12", + "ava": "5.1.0", + "koa-route": "^3.2.0", + "koa-websocket": "^7.0.0", + "query-string": "^8.1.0", + "ts-node": "^10.9.1", + "tslib": "^2.4.0", + "tsup": "^6.2.3", + "typescript": "^4.6.4", + "yargs": "^17.6.2" + } +} diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts new file mode 100644 index 000000000..3af58a3dc --- /dev/null +++ b/packages/lightning-mock/src/api-dev.ts @@ -0,0 +1,186 @@ +/* + * This module sets up a bunch of dev-only APIs + * These are not intended to be reflected in Lightning itself + */ +import Koa from 'koa'; +import Router from '@koa/router'; +import { Logger } from '@openfn/logger'; +import crypto from 'node:crypto'; + +import { Attempt } from './types'; +import { ServerState } from './server'; +import { ATTEMPT_COMPLETE, AttemptCompletePayload } from './events'; + +type LightningEvents = 'log' | 'attempt-complete'; + +type DataClip = any; + +export type DevApp = Koa & { + addCredential(id: string, cred: Credential): void; + addDataclip(id: string, data: DataClip): void; + enqueueAttempt(attempt: Attempt): void; + getAttempt(id: string): Attempt; + getCredential(id: string): Credential; + getDataclip(id: string): DataClip; + getQueueLength(): number; + getResult(attemptId: string): any; + getState(): ServerState; + on(event: LightningEvents, fn: (evt: any) => void): void; + once(event: LightningEvents, fn: (evt: any) => void): void; + onSocketEvent( + event: LightningEvents, + attemptId: string, + fn: (evt: any) => void + ): void; + registerAttempt(attempt: Attempt): void; + reset(): void; + startAttempt(id: string): any; + waitForResult(attemptId: string): Promise; +}; + +type Api = { + startAttempt(attemptId: string): void; +}; + +const setupDevAPI = ( + app: DevApp, + state: ServerState, + logger: Logger, + api: Api +) => { + // Dev APIs for unit testing + app.addCredential = (id: string, cred: Credential) => { + logger.info(`Add credential ${id}`); + state.credentials[id] = cred; + }; + + app.getCredential = (id: string) => state.credentials[id]; + + app.addDataclip = (id: string, data: any) => { + logger.info(`Add dataclip ${id}`); + state.dataclips[id] = data; + }; + + app.getDataclip = (id: string) => state.dataclips[id]; + + app.enqueueAttempt = (attempt: Attempt, workerId = 'rtm') => { + state.attempts[attempt.id] = attempt; + state.results[attempt.id] = { + workerId, // TODO + state: null, + }; + state.pending[attempt.id] = { + status: 'queued', + logs: [], + }; + state.queue.push(attempt.id); + }; + + app.getAttempt = (id: string) => state.attempts[id]; + + app.getState = () => state; + + // Promise which returns when a workflow is complete + app.waitForResult = (attemptId: string) => { + return new Promise((resolve) => { + const handler = (evt: { + payload: AttemptCompletePayload; + attemptId: string; + _state: ServerState; + dataclip: any; + }) => { + if (evt.attemptId === attemptId) { + state.events.removeListener(ATTEMPT_COMPLETE, handler); + const result = state.dataclips[evt.payload.final_dataclip_id]; + resolve(result); + } + }; + state.events.addListener(ATTEMPT_COMPLETE, handler); + }); + }; + + app.reset = () => { + state.queue = []; + state.results = {}; + }; + + app.getQueueLength = () => state.queue.length; + + app.getResult = (attemptId: string) => state.results[attemptId]?.state; + + app.startAttempt = (attemptId: string) => api.startAttempt(attemptId); + + // TODO probably remove? + app.registerAttempt = (attempt: any) => { + state.attempts[attempt.id] = attempt; + }; + + // TODO these are overriding koa's event handler - should I be doing something different? + + // @ts-ignore + app.on = (event: LightningEvents, fn: (evt: any) => void) => { + state.events.addListener(event, fn); + }; + + // @ts-ignore + app.once = (event: LightningEvents, fn: (evt: any) => void) => { + state.events.once(event, fn); + }; + + app.onSocketEvent = ( + event: LightningEvents, + attemptId: string, + fn: (evt: any) => void + ) => { + function handler(e: any) { + if (e.attemptId && e.attemptId === attemptId) { + state.events.removeListener(event, handler); + fn(e); + } + } + state.events.addListener(event, handler); + }; +}; + +// Set up some rest endpoints +// Note that these are NOT prefixed +const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { + const router = new Router(); + + router.post('/attempt', (ctx) => { + const attempt = ctx.request.body as Attempt; + + if (!attempt) { + ctx.response.status = 400; + return; + } + + logger.info('Adding new attempt to queue:', attempt.id); + logger.debug(attempt); + + if (!attempt.id) { + attempt.id = crypto.randomUUID(); + logger.info('Generating new id for incoming attempt:', attempt.id); + } + + // convert credentials and dataclips + attempt.jobs.forEach((job) => { + if (job.credential) { + const cid = crypto.randomUUID(); + state.credentials[cid] = job.credential; + job.credential = cid; + } + }); + + app.enqueueAttempt(attempt); + + ctx.response.status = 200; + }); + + return router.routes(); +}; + +export default (app: DevApp, state: ServerState, logger: Logger, api: Api) => { + setupDevAPI(app, state, logger, api); + return setupRestAPI(app, state, logger); +}; diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts new file mode 100644 index 000000000..7c05aad07 --- /dev/null +++ b/packages/lightning-mock/src/api-sockets.ts @@ -0,0 +1,335 @@ +import { WebSocketServer } from 'ws'; +import createLogger, { Logger } from '@openfn/logger'; + +import type { ServerState } from './server'; + +import { extractAttemptId } from './util'; + +import createPheonixMockSocketServer, { + DevSocket, + PhoenixEvent, +} from './socket-server'; +import { + ATTEMPT_COMPLETE, + AttemptCompletePayload, + AttemptCompleteReply, + ATTEMPT_LOG, + AttemptLogPayload, + AttemptLogReply, + CLAIM, + ClaimPayload, + ClaimReply, + ClaimAttempt, + GET_ATTEMPT, + GetAttemptPayload, + GetAttemptReply, + GET_CREDENTIAL, + GetCredentialPayload, + GetCredentialReply, + GET_DATACLIP, + GetDataclipPayload, + GetDataClipReply, + RUN_COMPLETE, + RunCompletePayload, + RUN_START, + RunStart, + RunStartReply, + RunCompleteReply, +} from './events'; + +import type { Server } from 'http'; +import { stringify } from './util'; + +// dumb cloning id +// just an idea for unit tests +const clone = (state: ServerState) => { + const { events, ...rest } = state; + return JSON.parse(JSON.stringify(rest)); +}; + +const enc = new TextEncoder(); + +// this new API is websocket based +// Events map to handlers +// can I even implement this in JS? Not with pheonix anyway. hmm. +// dead at the first hurdle really. +// what if I do the server side mock in koa, can I use the pheonix client to connect? +const createSocketAPI = ( + state: ServerState, + path: string, + httpServer: Server, + logger?: Logger +) => { + // set up a websocket server to listen to connections + const server = new WebSocketServer({ + server: httpServer, + + // Note: phoenix websocket will connect to /websocket + path: path ? `${path}/websocket` : undefined, + }); + + // pass that through to the phoenix mock + const wss = createPheonixMockSocketServer({ + // @ts-ignore server typings + server, + state, + logger: logger && createLogger('PHX', { level: 'debug' }), + }); + + wss.registerEvents('worker:queue', { + [CLAIM]: (ws, event: PhoenixEvent) => { + const { attempts } = pullClaim(state, ws, event); + attempts.forEach((attempt) => { + state.events.emit(CLAIM, { + attemptId: attempt.id, + payload: attempt, + state: clone(state), + }); + }); + }, + }); + + const startAttempt = (attemptId: string) => { + logger && logger.debug(`joining channel attempt:${attemptId}`); + + // mark the attempt as started on the server + state.pending[attemptId] = { + status: 'started', + logs: [], + }; + + const wrap = ( + handler: ( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent, + attemptId: string + ) => void + ) => { + return (ws: DevSocket, event: PhoenixEvent) => { + handler(state, ws, event, attemptId); + // emit each event and the state after to the event handler, for debug + state.events.emit(event.event, { + attemptId, + payload: event.payload, + state: clone(state), + }); + }; + }; + + const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { + [GET_ATTEMPT]: wrap(getAttempt), + [GET_CREDENTIAL]: wrap(getCredential), + [GET_DATACLIP]: wrap(getDataclip), + [RUN_START]: wrap(handleRunStart), + [ATTEMPT_LOG]: wrap(handleLog), + [RUN_COMPLETE]: wrap(handleRunComplete), + [ATTEMPT_COMPLETE]: wrap((...args) => { + handleAttemptComplete(...args); + unsubscribe(); + }), + }); + }; + + return { + startAttempt, + }; + + // pull claim will try and pull a claim off the queue, + // and reply with the response + // the reply ensures that only the calling worker will get the attempt + function pullClaim( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic } = evt; + const { queue } = state; + let count = 1; + + const attempts: ClaimAttempt[] = []; + const payload = { + status: 'ok' as const, + response: { attempts } as ClaimReply, + }; + + while (count > 0 && queue.length) { + // TODO assign the worker id to the attempt + // Not needed by the mocks at the moment + const next = queue.shift(); + // TODO the token in the mock is trivial because we're not going to do any validation on it yet + // TODO need to save the token associated with this attempt + attempts.push({ id: next!, token: 'x.y.z' }); + count -= 1; + + startAttempt(next!); + } + if (attempts.length) { + logger?.info(`Claiming ${attempts.length} attempts`); + } else { + logger?.info('No claims (queue empty)'); + } + + ws.reply({ ref, join_ref, topic, payload }); + return payload.response; + } + + function getAttempt( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic } = evt; + const attemptId = extractAttemptId(topic); + const response = state.attempts[attemptId]; + + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + response, + }, + }); + } + + function getCredential( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic, payload } = evt; + const response = state.credentials[payload.id]; + // console.log(topic, event, response); + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + response, + }, + }); + } + + // TODO this mock function is broken in the phoenix package update + // (I am not TOO worried, the actual integration works fine) + function getDataclip( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, topic, join_ref } = evt; + const dataclip = state.dataclips[evt.payload.id]; + + // Send the data as an ArrayBuffer (our stringify function will do this) + const payload = { + status: 'ok', + response: enc.encode(stringify(dataclip)), + }; + + ws.reply({ + ref, + join_ref, + topic, + // @ts-ignore + payload, + }); + } + + function handleLog( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic, payload } = evt; + const { attempt_id: attemptId } = payload; + + state.pending[attemptId].logs.push(payload); + + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + }, + }); + } + + function handleAttemptComplete( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent, + attemptId: string + ) { + const { ref, join_ref, topic, payload } = evt; + const { final_dataclip_id } = payload; + + logger?.info('Completed attempt ', attemptId); + logger?.debug(final_dataclip_id); + + state.pending[attemptId].status = 'complete'; + if (!state.results[attemptId]) { + state.results[attemptId] = { state: null, workerId: 'mock' }; + } + state.results[attemptId].state = state.dataclips[final_dataclip_id]; + + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + }, + }); + } + + function handleRunStart( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic } = evt; + if (!state.dataclips) { + state.dataclips = {}; + } + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + }, + }); + } + + function handleRunComplete( + state: ServerState, + ws: DevSocket, + evt: PhoenixEvent + ) { + const { ref, join_ref, topic } = evt; + const { output_dataclip_id, output_dataclip } = evt.payload; + + if (output_dataclip_id) { + if (!state.dataclips) { + state.dataclips = {}; + } + state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip!); + } + + // be polite and acknowledge the event + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'ok', + }, + }); + } +}; + +export default createSocketAPI; diff --git a/packages/lightning-mock/src/events.ts b/packages/lightning-mock/src/events.ts new file mode 100644 index 000000000..4aca14ce4 --- /dev/null +++ b/packages/lightning-mock/src/events.ts @@ -0,0 +1,68 @@ +import { Attempt } from './types'; + +/** + * These events are copied out of ws-worker + * There is a danger of them diverging + */ + +export const CLAIM = 'claim'; + +export type ClaimPayload = { demand?: number }; +export type ClaimReply = { attempts: Array }; +export type ClaimAttempt = { id: string; token: string }; + +export const GET_ATTEMPT = 'fetch:attempt'; +export type GetAttemptPayload = void; // no payload +export type GetAttemptReply = Attempt; + +export const GET_CREDENTIAL = 'fetch:credential'; +export type GetCredentialPayload = { id: string }; +// credential in-line, no wrapper, arbitrary data +export type GetCredentialReply = {}; + +export const GET_DATACLIP = 'fetch:dataclip'; +export type GetDataclipPayload = { id: string }; +export type GetDataClipReply = Uint8Array; // represents a json string Attempt + +export const ATTEMPT_START = 'attempt:start'; // attemptId, timestamp +export type AttemptStartPayload = void; // no payload +export type AttemptStartReply = void; // no payload + +export const ATTEMPT_COMPLETE = 'attempt:complete'; // attemptId, timestamp, result, stats +export type AttemptCompletePayload = { + final_dataclip_id: string; + status: 'success' | 'fail' | 'crash' | 'timeout'; + stats?: any; +}; // TODO dataclip -> result? output_dataclip? +export type AttemptCompleteReply = undefined; + +export const ATTEMPT_LOG = 'attempt:log'; // level, namespace (job,runtime,adaptor), message, time +export type AttemptLogPayload = { + message: Array; + timestamp: number; + attempt_id: string; + level?: string; + source?: string; // namespace + job_id?: string; + run_id?: string; +}; +export type AttemptLogReply = void; + +export const RUN_START = 'run:start'; +export type RunStart = { + job_id: string; + run_id: string; + attempt_id?: string; + input_dataclip_id?: string; //hmm +}; +export type RunStartReply = void; + +export const RUN_COMPLETE = 'run:complete'; +export type RunCompletePayload = { + attempt_id?: string; + job_id: string; + run_id: string; + output_dataclip?: string; + output_dataclip_id?: string; +}; +export type RunCompleteReply = void; diff --git a/packages/lightning-mock/src/index.ts b/packages/lightning-mock/src/index.ts new file mode 100644 index 000000000..5f3b83d4f --- /dev/null +++ b/packages/lightning-mock/src/index.ts @@ -0,0 +1,2 @@ +import createLightningServer from './server'; +export default createLightningServer; diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts new file mode 100644 index 000000000..49dd217ca --- /dev/null +++ b/packages/lightning-mock/src/server.ts @@ -0,0 +1,99 @@ +import { EventEmitter } from 'node:events'; +import Koa from 'koa'; +import bodyParser from 'koa-bodyparser'; +import koaLogger from 'koa-logger'; +import createLogger, { + createMockLogger, + LogLevel, + Logger, +} from '@openfn/logger'; + +import createWebSocketAPI from './api-sockets'; +import createDevAPI from './api-dev'; +import { Attempt } from './types'; +import { AttemptLogPayload } from './events'; + +export type AttemptState = { + status: 'queued' | 'started' | 'complete'; + logs: AttemptLogPayload[]; +}; + +export type ServerState = { + queue: AttemptId[]; + + // list of credentials by id + credentials: Record; + + // list of attempts by id + attempts: Record; + + // list of dataclips by id + dataclips: Record; + + // Tracking state of known attempts + // TODO include the engine id and token + pending: Record; + + // Track all completed attempts here + results: Record; + + // event emitter for debugging and observability + events: EventEmitter; +}; + +export type LightningOptions = { + logger?: Logger; + logLevel?: LogLevel; + port?: string | number; +}; + +export type AttemptId = string; + +// a mock lightning server +const createLightningServer = (options: LightningOptions = {}) => { + const logger = options.logger || createMockLogger(); + + const state = { + credentials: {}, + attempts: {}, + dataclips: {}, + pending: {}, + + queue: [] as AttemptId[], + results: {}, + events: new EventEmitter(), + } as ServerState; + + const app = new Koa(); + app.use(bodyParser()); + + const port = options.port || 8888; + const server = app.listen(port); + logger.info('Listening on ', port); + + // Only create a http logger if there's a top-level logger passed + // This is a bit flaky really but whatever + if (options.logger) { + const httpLogger = createLogger('HTTP', { level: 'debug' }); + const klogger = koaLogger((str) => httpLogger.debug(str)); + app.use(klogger); + } + + // Setup the websocket API + const api = createWebSocketAPI( + state, + '/worker', // TODO I should option drive this + server, + options.logger && logger + ); + + app.use(createDevAPI(app as any, state, logger, api)); + + (app as any).destroy = () => { + server.close(); + }; + + return app; +}; + +export default createLightningServer; diff --git a/packages/lightning-mock/src/socket-server.ts b/packages/lightning-mock/src/socket-server.ts new file mode 100644 index 000000000..a34069637 --- /dev/null +++ b/packages/lightning-mock/src/socket-server.ts @@ -0,0 +1,268 @@ +/** + * This module creates a mock pheonix socket server + * It uses a standard ws server but wraps messages up in a + * structure that pheonix sockets can understand + * It also adds some dev and debug APIs, useful for unit testing + */ +import { WebSocketServer, WebSocket } from 'ws'; +import querystring from 'query-string'; +// @ts-ignore +import { Serializer } from 'phoenix'; + +import { ATTEMPT_PREFIX, extractAttemptId } from './util'; +import { ServerState } from './server'; +import { stringify } from './util'; + +import type { Logger } from '@openfn/logger'; + +type Topic = string; + +const decoder = Serializer.decode.bind(Serializer); +const decode = (data: any) => new Promise((done) => decoder(data, done)); + +const encoder = Serializer.encode.bind(Serializer); +const encode = (data: any) => + new Promise((done) => { + if (data.payload?.response && data.payload.response instanceof Uint8Array) { + // special encoding logic if the payload is a buffer + // (we need to do this for dataclips) + data.payload.response = Array.from(data.payload.response); + } + encoder(data, done); + }); + +export type PhoenixEventStatus = 'ok' | 'error' | 'timeout'; + +// websocket with a couple of dev-friendly APIs +export type DevSocket = WebSocket & { + reply: (evt: PhoenixReply) => void; + sendJSON: ({ event, topic, ref }: PhoenixEvent) => void; +}; + +export type PhoenixEvent

= { + topic: Topic; + event: string; + payload: P; + ref: string; + join_ref: string; +}; + +export type PhoenixReply = { + topic: Topic; + payload: { + status: PhoenixEventStatus; + response?: R; + }; + ref: string; + join_ref: string; +}; + +type EventHandler = (ws: DevSocket, event: PhoenixEvent) => void; + +type CreateServerOptions = { + port?: number; + server: typeof WebSocketServer; + state: ServerState; + logger?: Logger; + onMessage?: (evt: PhoenixEvent) => void; +}; + +type MockSocketServer = typeof WebSocketServer & { + // Dev/debug APIs + listenToChannel: ( + topic: Topic, + fn: EventHandler + ) => { unsubscribe: () => void }; + waitForMessage: (topic: Topic, event: string) => Promise; + registerEvents: ( + topic: Topic, + events: Record + ) => { unsubscribe: () => void }; +}; + +function createServer({ + port = 8080, + server, + state, + logger, + onMessage = () => {}, +}: CreateServerOptions) { + const channels: Record> = { + // create a stub listener for pheonix to prevent errors + phoenix: new Set([() => null]), + }; + + const wsServer = + server || + new WebSocketServer({ + port, + }); + + if (!server) { + logger?.info('pheonix mock websocket server listening on', port); + } + + const events = { + // When joining a channel, we need to send a chan_reply_{ref} message back to the socket + phx_join: ( + ws: DevSocket, + { topic, ref, payload, join_ref }: PhoenixEvent + ) => { + let status: PhoenixEventStatus = 'ok'; + let response = 'ok'; + + // Validation on attempt:* channels + // TODO is this logic in the right place? + if (topic.startsWith(ATTEMPT_PREFIX)) { + const attemptId = extractAttemptId(topic); + if (!state.pending[attemptId]) { + status = 'error'; + response = 'invalid_attempt_id'; + } else if (!payload.token) { + // TODO better token validation here + status = 'error'; + response = 'invalid_token'; + } + } + ws.reply({ + topic, + payload: { status, response }, + ref, + join_ref, + }); + }, + }; + + // @ts-ignore something wierd about the wsServer typing + wsServer.on('connection', function (ws: DevSocket, req: any) { + logger?.info('new client connected'); + + // Ensure that a JWT token is added to the + const [_path, query] = req.url.split('?'); + const { token } = querystring.parse(query); + + // TODO for now, there's no validation on the token in this mock + + // If there is no token (or later, if invalid), close the connection immediately + if (!token) { + logger?.error('INVALID TOKEN'); + ws.close(); + + // TODO I'd love to send back a 403 here, not sure how to do it + // (and it's not important in the mock really) + return; + } + + ws.reply = async ({ + ref, + topic, + payload, + join_ref, + }: PhoenixReply) => { + // TODO only stringify payload if not a buffer + logger?.debug(`<< [${topic}] chan_reply_${ref} ` + stringify(payload)); + const evt = await encode({ + event: `chan_reply_${ref}`, + ref, + join_ref, + topic, + payload, + }); + // @ts-ignore + ws.send(evt); + }; + + ws.sendJSON = async ({ event, ref, topic, payload }: PhoenixEvent) => { + logger?.debug(`<< [${topic}] ${event} ` + stringify(payload)); + const evt = await encode({ + event, + ref, + topic, + payload: stringify(payload), // TODO do we stringify this? All of it? + }); + // @ts-ignore + ws.send(evt); + }; + + ws.on('message', async function (data: string) { + // decode the data + const evt = (await decode(data)) as PhoenixEvent; + onMessage(evt); + + if (evt.topic) { + // phx sends this info in each message + const { topic, event, payload, ref, join_ref } = evt; + + logger?.debug(`>> [${topic}] ${event} ${ref} :: ${stringify(payload)}`); + + if (event in events) { + // handle system/phoenix events + // @ts-ignore + events[event](ws, { topic, payload, ref, join_ref }); + } else { + // handle custom/user events + if (channels[topic] && channels[topic].size) { + channels[topic].forEach((fn) => { + fn(ws, { event, topic, payload, ref, join_ref }); + }); + } else { + // This behaviour is just a convenience for unit tesdting + ws.reply({ + ref, + join_ref, + topic, + payload: { + status: 'error', + response: `There are no listeners on channel ${topic}`, + }, + }); + } + } + } + }); + }); + + const mockServer = wsServer as MockSocketServer; + + // debug API + // TODO should this in fact be (topic, event, fn)? + mockServer.listenToChannel = (topic: Topic, fn: EventHandler) => { + if (!channels[topic]) { + channels[topic] = new Set(); + } + + channels[topic].add(fn); + + return { + unsubscribe: () => { + channels[topic].delete(fn); + }, + }; + }; + + mockServer.waitForMessage = (topic: Topic, event: string) => { + return new Promise((resolve) => { + const listener = mockServer.listenToChannel(topic, (_ws, e) => { + if (e.event === event) { + listener.unsubscribe(); + resolve(e); + } + }); + }); + }; + + mockServer.registerEvents = (topic: Topic, events) => { + // listen to all events in the channel + return mockServer.listenToChannel(topic, (ws, evt) => { + const { event } = evt; + // call the correct event handler for this event + if (events[event]) { + events[event](ws, evt); + } + }); + }; + + return mockServer as MockSocketServer; +} + +export default createServer; diff --git a/packages/lightning-mock/src/start.ts b/packages/lightning-mock/src/start.ts new file mode 100644 index 000000000..557d7b07f --- /dev/null +++ b/packages/lightning-mock/src/start.ts @@ -0,0 +1,29 @@ +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import createLogger from '@openfn/logger'; + +import createLightningServer from './server'; + +type Args = { + _: string[]; + port?: number; +}; + +const args = yargs(hideBin(process.argv)) + .command('server', 'Start a mock lighting server') + .option('port', { + alias: 'p', + description: 'Port to run the server on', + type: 'number', + default: 8888, + }) + .parse() as Args; + +const logger = createLogger('LNG', { level: 'debug' }); + +createLightningServer({ + port: args.port, + logger, +}); + +logger.success('Started mock Lightning server on ', args.port); diff --git a/packages/lightning-mock/src/types.ts b/packages/lightning-mock/src/types.ts new file mode 100644 index 000000000..f77948a64 --- /dev/null +++ b/packages/lightning-mock/src/types.ts @@ -0,0 +1,32 @@ +export type Node = { + id: string; + body?: string; + adaptor?: string; + credential?: any; // TODO tighten this up, string or object + type?: 'webhook' | 'cron'; // trigger only + state?: any; // Initial state / defaults +}; + +export interface Edge { + id: string; + source_job_id?: string; + source_trigger_id?: string; + target_job_id: string; + name?: string; + condition?: string; + error_path?: boolean; + errors?: any; +} + +// An attempt object returned by Lightning +export type Attempt = { + id: string; + dataclip_id: string; + starting_node_id: string; + + triggers: Node[]; + jobs: Node[]; + edges: Edge[]; + + options?: Record; // TODO type the expected options +}; diff --git a/packages/lightning-mock/src/util.ts b/packages/lightning-mock/src/util.ts new file mode 100644 index 000000000..bfc728b5f --- /dev/null +++ b/packages/lightning-mock/src/util.ts @@ -0,0 +1,15 @@ +import fss from 'fast-safe-stringify'; + +export const ATTEMPT_PREFIX = 'attempt:'; + +export const extractAttemptId = (topic: string) => + topic.substr(ATTEMPT_PREFIX.length); + +// This is copied out of ws-worker and untested here +export const stringify = (obj: any): string => + fss(obj, (_key: string, value: any) => { + if (value instanceof Uint8Array) { + return Array.from(value); + } + return value; + }); diff --git a/packages/lightning-mock/test/data.ts b/packages/lightning-mock/test/data.ts new file mode 100644 index 000000000..e6e773be6 --- /dev/null +++ b/packages/lightning-mock/test/data.ts @@ -0,0 +1,32 @@ +export const credentials = { + a: { + user: 'bobby', + password: 'password1', + }, +}; + +export const dataclips = { + d: { + count: 1, + }, +}; + +export const attempts = { + 'attempt-1': { + id: 'attempt-1', + // TODO how should this be structure? + input: { + data: 'd', + }, + triggers: [], + edges: [], + jobs: [ + { + id: 'job-1', + adaptor: '@openfn/language-common@1.0.0', + body: 'fn(a => a)', + credential: 'a', + }, + ], + }, +}; diff --git a/packages/lightning-mock/test/lightning.test.ts b/packages/lightning-mock/test/lightning.test.ts new file mode 100644 index 000000000..60b5cc798 --- /dev/null +++ b/packages/lightning-mock/test/lightning.test.ts @@ -0,0 +1,345 @@ +import test from 'ava'; +import createLightningServer from '../src/server'; + +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; + +import { attempts, credentials, dataclips } from './data'; +import { + ATTEMPT_COMPLETE, + AttemptCompletePayload, + ATTEMPT_LOG, + CLAIM, + GET_ATTEMPT, + GET_CREDENTIAL, + GET_DATACLIP, +} from '../src/events'; +import type { Attempt } from '../src/types'; +import { JSONLog } from '@openfn/logger'; + +const endpoint = 'ws://localhost:7777/worker'; + +const enc = new TextDecoder('utf-8'); + +type Channel = any; + +let server; +let client; + +// Set up a lightning server and a phoenix socket client before each test +test.before( + () => + new Promise((done) => { + server = createLightningServer({ port: 7777 }); + + // Note that we need a token to connect, but the mock here + // doesn't (yet) do any validation on that token + client = new Socket(endpoint, { + params: { token: 'x.y.z' }, + timeout: 1000 * 120, + transport: WebSocket, + }); + client.onOpen(done); + client.connect(); + }) +); + +test.afterEach(() => { + server.reset(); +}); + +test.after(() => { + server.destroy(); +}); + +const attempt1 = attempts['attempt-1']; + +const join = (channelName: string, params: any = {}): Promise => + new Promise((done, reject) => { + const channel = client.channel(channelName, params); + channel + .join() + .receive('ok', () => { + done(channel); + }) + .receive('error', (err) => { + // err will be the response message on the payload (ie, invalid_token, invalid_attempt_id etc) + reject(new Error(err)); + }); + }); + +// Test some dev hooks +// enqueue attempt should add id to the queue and register the state, credentials and body +test.serial('should setup an attempt at /POST /attempt', async (t) => { + const state = server.getState(); + + t.is(Object.keys(state.credentials).length, 0); + t.is(Object.keys(state.attempts).length, 0); + t.is(Object.keys(state.attempts).length, 0); + + const attempt: Attempt = { + id: 'a', + triggers: [], + jobs: [ + { + body: 'abc', + dataclip: {}, // not sure how this will work on the attempt yet + credential: { + // this will be converted into a string for lazy loading + user: 'john', + password: 'rambo', + }, + }, + ], + edges: [], + }; + + await fetch('http://localhost:7777/attempt', { + method: 'POST', + body: JSON.stringify(attempt), + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + }); + + const newState = server.getState(); + t.is(Object.keys(newState.attempts).length, 1); + const a = server.getAttempt('a'); + t.truthy(a); + t.is(server.getQueueLength(), 1); + + t.is(Object.keys(newState.credentials).length, 1); + + const job = a.jobs[0]; + t.assert(typeof job.credential === 'string'); + const c = server.getCredential(job.credential); + t.is(c.user, 'john'); +}); + +test.serial('provide a phoenix websocket at /worker', (t) => { + // client should be connected before this test runs + t.is(client.connectionState(), 'open'); +}); + +test.serial('reject ws connections without a token', (t) => { + return new Promise((done) => { + // client should be connected before this test runs + const socket = new Socket(endpoint, { transport: WebSocket }); + socket.onClose(() => { + t.pass(); + done(); + }); + socket.connect(); + }); +}); + +test.serial('respond to channel join requests', (t) => { + return new Promise(async (done, reject) => { + const channel = client.channel('x', {}); + + channel.join().receive('ok', (res) => { + t.is(res, 'ok'); + done(); + }); + }); +}); + +// TODO: only allow authorised workers to join workers +// TODO: only allow authorised attemtps to join an attempt channel + +test.serial( + 'claim attempt: reply for zero items if queue is empty', + (t) => + new Promise(async (done) => { + t.is(server.getQueueLength(), 0); + + const channel = await join('worker:queue'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + const { attempts } = response; + t.assert(Array.isArray(attempts)); + t.is(attempts.length, 0); + + t.is(server.getQueueLength(), 0); + done(); + }); + }) +); + +test.serial( + "claim attempt: reply with an attempt id if there's an attempt in the queue", + (t) => + new Promise(async (done) => { + server.enqueueAttempt(attempt1); + t.is(server.getQueueLength(), 1); + + const channel = await join('worker:queue'); + + // response is an array of attempt ids + channel.push(CLAIM).receive('ok', (response) => { + const { attempts } = response; + t.truthy(attempts); + t.is(attempts.length, 1); + t.deepEqual(attempts[0], { id: 'attempt-1', token: 'x.y.z' }); + + // ensure the server state has changed + t.is(server.getQueueLength(), 0); + done(); + }); + }) +); + +// TODO is it even worth doing this? Easier for a socket to pull one at a time? +// It would also ensure better distribution if 10 workers ask at the same time, they'll get +// one each then come back for more +test.todo('claim attempt: reply with multiple attempt ids'); + +test.serial('create a channel for an attempt', async (t) => { + server.startAttempt('wibble'); + await join('attempt:wibble', { token: 'a.b.c' }); + t.pass('connection ok'); +}); + +test.serial('do not allow to join a channel without a token', async (t) => { + server.startAttempt('wibble'); + await t.throwsAsync(() => join('attempt:wibble'), { + message: 'invalid_token', + }); +}); + +test.todo('do not allow to join a channel without a valid token'); + +test.serial('reject channels for attempts that are not started', async (t) => { + await t.throwsAsync(() => join('attempt:xyz'), { + message: 'invalid_attempt_id', + }); +}); + +test.serial('get attempt data through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.registerAttempt(attempt1); + server.startAttempt(attempt1.id); + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(GET_ATTEMPT).receive('ok', (attempt) => { + t.deepEqual(attempt, attempt1); + done(); + }); + }); +}); + +test.serial('complete an attempt through the attempt channel', async (t) => { + return new Promise(async (done) => { + const a = attempt1; + server.registerAttempt(a); + server.startAttempt(a.id); + server.addDataclip('abc', { answer: 42 }); + + const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); + channel + .push(ATTEMPT_COMPLETE, { final_dataclip_id: 'abc' }) + .receive('ok', () => { + const { pending, results } = server.getState(); + t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); + t.deepEqual(results[a.id].state, { answer: 42 }); + done(); + }); + }); +}); + +test.serial('logs are saved and acknowledged', async (t) => { + return new Promise(async (done) => { + server.registerAttempt(attempt1); + server.startAttempt(attempt1.id); + + const log = { + attempt_id: attempt1.id, + level: 'info', + name: 'R/T', + message: ['Did the thing'], + time: new Date().getTime(), + } as JSONLog; + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(ATTEMPT_LOG, log).receive('ok', () => { + const { pending } = server.getState(); + const [savedLog] = pending[attempt1.id].logs; + t.deepEqual(savedLog, log); + done(); + }); + }); +}); + +test.serial('unsubscribe after attempt complete', async (t) => { + return new Promise(async (done) => { + const a = attempt1; + server.registerAttempt(a); + server.startAttempt(a.id); + + const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); + channel.push(ATTEMPT_COMPLETE).receive('ok', () => { + // After the complete event, the listener should unsubscribe to the channel + // The mock will send an error to any unhandled events in that channel + channel.push(ATTEMPT_COMPLETE).receive('error', () => { + t.pass(); + done(); + }); + }); + }); +}); + +test.serial('get credential through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.startAttempt(attempt1.id); + server.addCredential('a', credentials['a']); + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(GET_CREDENTIAL, { id: 'a' }).receive('ok', (result) => { + t.deepEqual(result, credentials['a']); + done(); + }); + }); +}); + +test.serial('get dataclip through the attempt channel', async (t) => { + return new Promise(async (done) => { + server.startAttempt(attempt1.id); + server.addDataclip('d', dataclips['d']); + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(GET_DATACLIP, { id: 'd' }).receive('ok', (result) => { + const str = enc.decode(new Uint8Array(result)); + const dataclip = JSON.parse(str); + t.deepEqual(dataclip, dataclips['d']); + done(); + }); + }); +}); + +// TODO test that all events are proxied out to server.on + +test.serial( + 'waitForResult should return logs and dataclip when an attempt is completed', + async (t) => { + return new Promise(async (done) => { + const result = { answer: 42 }; + + server.startAttempt(attempt1.id); + server.addDataclip('result', result); + + server.waitForResult(attempt1.id).then((dataclip) => { + t.deepEqual(result, dataclip); + done(); + }); + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(ATTEMPT_COMPLETE, { + final_dataclip_id: 'result', + } as AttemptCompletePayload); + }); + } +); + +// test.serial('getLogs should return logs', async (t) => {}); diff --git a/packages/lightning-mock/tsconfig.json b/packages/lightning-mock/tsconfig.json new file mode 100644 index 000000000..ba1452256 --- /dev/null +++ b/packages/lightning-mock/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.common", + "include": ["src/**/*.ts"], + "compilerOptions": { + "module": "ESNext" + } +} From cd0f3f3c5570d1ff21d444ee8af7de45391c17a6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 11:39:03 +0100 Subject: [PATCH 150/232] lightning-mock: move over socket server tests --- packages/lightning-mock/package.json | 2 +- .../lightning-mock/test/socket-server.test.ts | 153 ++++++++++++++++++ 2 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 packages/lightning-mock/test/socket-server.test.ts diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 4a0652c55..352289b2f 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "description": "A mock Lightning server", - "main": "index.js", + "main": "dist/index.js", "type": "module", "scripts": { "test": "pnpm ava --serial", diff --git a/packages/lightning-mock/test/socket-server.test.ts b/packages/lightning-mock/test/socket-server.test.ts new file mode 100644 index 000000000..7636f13e2 --- /dev/null +++ b/packages/lightning-mock/test/socket-server.test.ts @@ -0,0 +1,153 @@ +import test from 'ava'; +import { Socket } from 'phoenix'; +import { WebSocket } from 'ws'; + +import createSocketServer from '../src/socket-server'; + +let socket; +let server; +let messages; + +const wait = (duration = 10) => + new Promise((resolve) => { + setTimeout(resolve, duration); + }); + +test.beforeEach( + () => + new Promise((done) => { + messages = []; + // @ts-ignore I don't care about missing server options here + server = createSocketServer({ + onMessage: (evt) => { + messages.push(evt); + }, + }); + + socket = new Socket('ws://localhost:8080', { + transport: WebSocket, + params: { token: 'x.y.z' }, + }); + + socket.onOpen(done); + + socket.connect(); + }) +); + +test.afterEach(() => { + server.close(); +}); + +test.serial('respond to connection join requests', async (t) => { + return new Promise((resolve) => { + const channel = socket.channel('x', {}); + + channel + .join() + .receive('ok', (resp) => { + t.is(resp, 'ok'); + + channel.push('hello'); + resolve(); + }) + .receive('error', (e) => { + console.log(e); + }); + }); +}); + +test.serial('send a message', async (t) => { + return new Promise((resolve) => { + const channel = socket.channel('x', {}); + + server.listenToChannel('x', (_ws, { payload, event }) => { + t.is(event, 'hello'); + t.deepEqual(payload, { x: 1 }); + + resolve(); + }); + + channel.join(); + + channel.push('hello', { x: 1 }); + }); +}); + +test.serial('send a message only to one channel', async (t) => { + let didCallX = false; + let didCallY = false; + + const x = socket.channel('x', {}); + x.join(); + + const y = socket.channel('y', {}); + y.join(); + + server.listenToChannel('x', () => { + didCallX = true; + }); + server.listenToChannel('y', () => { + didCallY = true; + }); + + x.push('hello', { x: 1 }); + + await wait(); + + t.true(didCallX); + t.false(didCallY); +}); + +test.serial('unsubscribe', (t) => { + return new Promise(async (resolve) => { + let count = 0; + + const channel = socket.channel('x', {}); + channel.join(); + + const listener = server.listenToChannel('x', () => { + count++; + }); + + channel.push('hello', { x: 1 }); + await wait(100); + + t.is(count, 1); + + listener.unsubscribe(); + + channel.push('hello', { x: 1 }); + await wait(); + + t.is(count, 1); + + resolve(); + }); +}); + +test.serial('wait for message', async (t) => { + const channel = socket.channel('x', {}); + channel.join(); + + channel.push('hello', { x: 1 }); + + const result = await server.waitForMessage('x', 'hello'); + t.truthy(result); +}); + +test.serial('onMessage', (t) => { + return new Promise((done) => { + const channel = socket.channel('x', {}); + channel.join().receive('ok', async () => { + t.is(messages.length, 1); + t.is(messages[0].event, 'phx_join'); + + channel.push('hello', { x: 1 }); + await server.waitForMessage('x', 'hello'); + t.is(messages.length, 2); + t.is(messages[1].event, 'hello'); + done(); + }); + }); +}); From 397b0a988c4684e6e9a3ba38bc7808abae928bbc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 11:39:48 +0100 Subject: [PATCH 151/232] ws-worker: move lightning mock into a different package --- packages/ws-worker/package.json | 5 +- .../ws-worker/src/mock/lightning/api-dev.ts | 186 ---------- .../src/mock/lightning/api-sockets.ts | 335 ----------------- .../ws-worker/src/mock/lightning/index.ts | 2 - .../ws-worker/src/mock/lightning/server.ts | 99 ----- .../src/mock/lightning/socket-server.ts | 268 -------------- .../ws-worker/src/mock/lightning/start.ts | 29 -- packages/ws-worker/src/mock/lightning/util.ts | 4 - packages/ws-worker/src/start.ts | 29 +- packages/ws-worker/test/integration.test.ts | 2 +- .../ws-worker/test/mock/lightning.test.ts | 346 ------------------ .../ws-worker/test/mock/socket-server.test.ts | 149 -------- packages/ws-worker/test/server.test.ts | 68 ---- 13 files changed, 19 insertions(+), 1503 deletions(-) delete mode 100644 packages/ws-worker/src/mock/lightning/api-dev.ts delete mode 100644 packages/ws-worker/src/mock/lightning/api-sockets.ts delete mode 100644 packages/ws-worker/src/mock/lightning/index.ts delete mode 100644 packages/ws-worker/src/mock/lightning/server.ts delete mode 100644 packages/ws-worker/src/mock/lightning/socket-server.ts delete mode 100644 packages/ws-worker/src/mock/lightning/start.ts delete mode 100644 packages/ws-worker/src/mock/lightning/util.ts delete mode 100644 packages/ws-worker/test/mock/lightning.test.ts delete mode 100644 packages/ws-worker/test/mock/socket-server.test.ts delete mode 100644 packages/ws-worker/test/server.test.ts diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 786e9212a..7ea4369db 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -4,7 +4,6 @@ "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "index.js", "type": "module", - "private": true, "scripts": { "test": "pnpm ava --serial", "test:types": "pnpm tsc --noEmit --project tsconfig.json", @@ -12,7 +11,8 @@ "build:watch": "pnpm build --watch", "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm --transpile-only src/start.ts", "start:lightning": "ts-node-esm --transpile-only src/mock/lightning/start.ts", - "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'" + "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'", + "pack": "pnpm pack --pack-destination ../../dist" }, "author": "Open Function Group ", "license": "ISC", @@ -41,6 +41,7 @@ "@types/nodemon": "1.19.3", "@types/phoenix": "^1.6.2", "@types/yargs": "^17.0.12", + "@openfn/lightning-mock": "workspace:*", "ava": "5.1.0", "koa-route": "^3.2.0", "koa-websocket": "^7.0.0", diff --git a/packages/ws-worker/src/mock/lightning/api-dev.ts b/packages/ws-worker/src/mock/lightning/api-dev.ts deleted file mode 100644 index a0e3fad65..000000000 --- a/packages/ws-worker/src/mock/lightning/api-dev.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* - * This module sets up a bunch of dev-only APIs - * These are not intended to be reflected in Lightning itself - */ -import Koa from 'koa'; -import Router from '@koa/router'; -import { Logger } from '@openfn/logger'; -import crypto from 'node:crypto'; - -import { Attempt } from '../../types'; -import { ServerState } from './server'; -import { ATTEMPT_COMPLETE, ATTEMPT_COMPLETE_PAYLOAD } from '../../events'; - -type LightningEvents = 'log' | 'attempt-complete'; - -type DataClip = any; - -export type DevApp = Koa & { - addCredential(id: string, cred: Credential): void; - addDataclip(id: string, data: DataClip): void; - enqueueAttempt(attempt: Attempt): void; - getAttempt(id: string): Attempt; - getCredential(id: string): Credential; - getDataclip(id: string): DataClip; - getQueueLength(): number; - getResult(attemptId: string): any; - getState(): ServerState; - on(event: LightningEvents, fn: (evt: any) => void): void; - once(event: LightningEvents, fn: (evt: any) => void): void; - onSocketEvent( - event: LightningEvents, - attemptId: string, - fn: (evt: any) => void - ): void; - registerAttempt(attempt: Attempt): void; - reset(): void; - startAttempt(id: string): any; - waitForResult(attemptId: string): Promise; -}; - -type Api = { - startAttempt(attemptId: string): void; -}; - -const setupDevAPI = ( - app: DevApp, - state: ServerState, - logger: Logger, - api: Api -) => { - // Dev APIs for unit testing - app.addCredential = (id: string, cred: Credential) => { - logger.info(`Add credential ${id}`); - state.credentials[id] = cred; - }; - - app.getCredential = (id: string) => state.credentials[id]; - - app.addDataclip = (id: string, data: any) => { - logger.info(`Add dataclip ${id}`); - state.dataclips[id] = data; - }; - - app.getDataclip = (id: string) => state.dataclips[id]; - - app.enqueueAttempt = (attempt: Attempt, workerId = 'rtm') => { - state.attempts[attempt.id] = attempt; - state.results[attempt.id] = { - workerId, // TODO - state: null, - }; - state.pending[attempt.id] = { - status: 'queued', - logs: [], - }; - state.queue.push(attempt.id); - }; - - app.getAttempt = (id: string) => state.attempts[id]; - - app.getState = () => state; - - // Promise which returns when a workflow is complete - app.waitForResult = (attemptId: string) => { - return new Promise((resolve) => { - const handler = (evt: { - payload: ATTEMPT_COMPLETE_PAYLOAD; - attemptId: string; - _state: ServerState; - dataclip: any; - }) => { - if (evt.attemptId === attemptId) { - state.events.removeListener(ATTEMPT_COMPLETE, handler); - const result = state.dataclips[evt.payload.final_dataclip_id]; - resolve(result); - } - }; - state.events.addListener(ATTEMPT_COMPLETE, handler); - }); - }; - - app.reset = () => { - state.queue = []; - state.results = {}; - }; - - app.getQueueLength = () => state.queue.length; - - app.getResult = (attemptId: string) => state.results[attemptId]?.state; - - app.startAttempt = (attemptId: string) => api.startAttempt(attemptId); - - // TODO probably remove? - app.registerAttempt = (attempt: any) => { - state.attempts[attempt.id] = attempt; - }; - - // TODO these are overriding koa's event handler - should I be doing something different? - - // @ts-ignore - app.on = (event: LightningEvents, fn: (evt: any) => void) => { - state.events.addListener(event, fn); - }; - - // @ts-ignore - app.once = (event: LightningEvents, fn: (evt: any) => void) => { - state.events.once(event, fn); - }; - - app.onSocketEvent = ( - event: LightningEvents, - attemptId: string, - fn: (evt: any) => void - ) => { - function handler(e: any) { - if (e.attemptId && e.attemptId === attemptId) { - state.events.removeListener(event, handler); - fn(e); - } - } - state.events.addListener(event, handler); - }; -}; - -// Set up some rest endpoints -// Note that these are NOT prefixed -const setupRestAPI = (app: DevApp, state: ServerState, logger: Logger) => { - const router = new Router(); - - router.post('/attempt', (ctx) => { - const attempt = ctx.request.body as Attempt; - - if (!attempt) { - ctx.response.status = 400; - return; - } - - logger.info('Adding new attempt to queue:', attempt.id); - logger.debug(attempt); - - if (!attempt.id) { - attempt.id = crypto.randomUUID(); - logger.info('Generating new id for incoming attempt:', attempt.id); - } - - // convert credentials and dataclips - attempt.jobs.forEach((job) => { - if (job.credential) { - const cid = crypto.randomUUID(); - state.credentials[cid] = job.credential; - job.credential = cid; - } - }); - - app.enqueueAttempt(attempt); - - ctx.response.status = 200; - }); - - return router.routes(); -}; - -export default (app: DevApp, state: ServerState, logger: Logger, api: Api) => { - setupDevAPI(app, state, logger, api); - return setupRestAPI(app, state, logger); -}; diff --git a/packages/ws-worker/src/mock/lightning/api-sockets.ts b/packages/ws-worker/src/mock/lightning/api-sockets.ts deleted file mode 100644 index 12e90f475..000000000 --- a/packages/ws-worker/src/mock/lightning/api-sockets.ts +++ /dev/null @@ -1,335 +0,0 @@ -import { WebSocketServer } from 'ws'; -import createLogger, { Logger } from '@openfn/logger'; - -import type { ServerState } from './server'; - -import { extractAttemptId } from './util'; - -import createPheonixMockSocketServer, { - DevSocket, - PhoenixEvent, -} from './socket-server'; -import { - ATTEMPT_COMPLETE, - ATTEMPT_COMPLETE_PAYLOAD, - ATTEMPT_COMPLETE_REPLY, - ATTEMPT_LOG, - ATTEMPT_LOG_PAYLOAD, - ATTEMPT_LOG_REPLY, - CLAIM, - CLAIM_PAYLOAD, - CLAIM_REPLY, - CLAIM_ATTEMPT, - GET_ATTEMPT, - GET_ATTEMPT_PAYLOAD, - GET_ATTEMPT_REPLY, - GET_CREDENTIAL, - GET_CREDENTIAL_PAYLOAD, - GET_CREDENTIAL_REPLY, - GET_DATACLIP, - GET_DATACLIP_PAYLOAD, - GET_DATACLIP_REPLY, - RUN_COMPLETE, - RUN_COMPLETE_PAYLOAD, - RUN_START, - RUN_START_PAYLOAD, - RUN_START_REPLY, - RUN_COMPLETE_REPLY, -} from '../../events'; - -import type { Server } from 'http'; -import { stringify } from '../../util'; - -// dumb cloning id -// just an idea for unit tests -const clone = (state: ServerState) => { - const { events, ...rest } = state; - return JSON.parse(JSON.stringify(rest)); -}; - -const enc = new TextEncoder(); - -// this new API is websocket based -// Events map to handlers -// can I even implement this in JS? Not with pheonix anyway. hmm. -// dead at the first hurdle really. -// what if I do the server side mock in koa, can I use the pheonix client to connect? -const createSocketAPI = ( - state: ServerState, - path: string, - httpServer: Server, - logger?: Logger -) => { - // set up a websocket server to listen to connections - const server = new WebSocketServer({ - server: httpServer, - - // Note: phoenix websocket will connect to /websocket - path: path ? `${path}/websocket` : undefined, - }); - - // pass that through to the phoenix mock - const wss = createPheonixMockSocketServer({ - // @ts-ignore server typings - server, - state, - logger: logger && createLogger('PHX', { level: 'debug' }), - }); - - wss.registerEvents('worker:queue', { - [CLAIM]: (ws, event: PhoenixEvent) => { - const { attempts } = pullClaim(state, ws, event); - attempts.forEach((attempt) => { - state.events.emit(CLAIM, { - attemptId: attempt.id, - payload: attempt, - state: clone(state), - }); - }); - }, - }); - - const startAttempt = (attemptId: string) => { - logger && logger.debug(`joining channel attempt:${attemptId}`); - - // mark the attempt as started on the server - state.pending[attemptId] = { - status: 'started', - logs: [], - }; - - const wrap = ( - handler: ( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent, - attemptId: string - ) => void - ) => { - return (ws: DevSocket, event: PhoenixEvent) => { - handler(state, ws, event, attemptId); - // emit each event and the state after to the event handler, for debug - state.events.emit(event.event, { - attemptId, - payload: event.payload, - state: clone(state), - }); - }; - }; - - const { unsubscribe } = wss.registerEvents(`attempt:${attemptId}`, { - [GET_ATTEMPT]: wrap(getAttempt), - [GET_CREDENTIAL]: wrap(getCredential), - [GET_DATACLIP]: wrap(getDataclip), - [RUN_START]: wrap(handleRunStart), - [ATTEMPT_LOG]: wrap(handleLog), - [RUN_COMPLETE]: wrap(handleRunComplete), - [ATTEMPT_COMPLETE]: wrap((...args) => { - handleAttemptComplete(...args); - unsubscribe(); - }), - }); - }; - - return { - startAttempt, - }; - - // pull claim will try and pull a claim off the queue, - // and reply with the response - // the reply ensures that only the calling worker will get the attempt - function pullClaim( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic } = evt; - const { queue } = state; - let count = 1; - - const attempts: CLAIM_ATTEMPT[] = []; - const payload = { - status: 'ok' as const, - response: { attempts } as CLAIM_REPLY, - }; - - while (count > 0 && queue.length) { - // TODO assign the worker id to the attempt - // Not needed by the mocks at the moment - const next = queue.shift(); - // TODO the token in the mock is trivial because we're not going to do any validation on it yet - // TODO need to save the token associated with this attempt - attempts.push({ id: next!, token: 'x.y.z' }); - count -= 1; - - startAttempt(next!); - } - if (attempts.length) { - logger?.info(`Claiming ${attempts.length} attempts`); - } else { - logger?.info('No claims (queue empty)'); - } - - ws.reply({ ref, join_ref, topic, payload }); - return payload.response; - } - - function getAttempt( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic } = evt; - const attemptId = extractAttemptId(topic); - const response = state.attempts[attemptId]; - - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - response, - }, - }); - } - - function getCredential( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic, payload } = evt; - const response = state.credentials[payload.id]; - // console.log(topic, event, response); - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - response, - }, - }); - } - - // TODO this mock function is broken in the phoenix package update - // (I am not TOO worried, the actual integration works fine) - function getDataclip( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, topic, join_ref } = evt; - const dataclip = state.dataclips[evt.payload.id]; - - // Send the data as an ArrayBuffer (our stringify function will do this) - const payload = { - status: 'ok', - response: enc.encode(stringify(dataclip)), - }; - - ws.reply({ - ref, - join_ref, - topic, - // @ts-ignore - payload, - }); - } - - function handleLog( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic, payload } = evt; - const { attempt_id: attemptId } = payload; - - state.pending[attemptId].logs.push(payload); - - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - }, - }); - } - - function handleAttemptComplete( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent, - attemptId: string - ) { - const { ref, join_ref, topic, payload } = evt; - const { final_dataclip_id } = payload; - - logger?.info('Completed attempt ', attemptId); - logger?.debug(final_dataclip_id); - - state.pending[attemptId].status = 'complete'; - if (!state.results[attemptId]) { - state.results[attemptId] = { state: null, workerId: 'mock' }; - } - state.results[attemptId].state = state.dataclips[final_dataclip_id]; - - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - }, - }); - } - - function handleRunStart( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic } = evt; - if (!state.dataclips) { - state.dataclips = {}; - } - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - }, - }); - } - - function handleRunComplete( - state: ServerState, - ws: DevSocket, - evt: PhoenixEvent - ) { - const { ref, join_ref, topic } = evt; - const { output_dataclip_id, output_dataclip } = evt.payload; - - if (output_dataclip_id) { - if (!state.dataclips) { - state.dataclips = {}; - } - state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip!); - } - - // be polite and acknowledge the event - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'ok', - }, - }); - } -}; - -export default createSocketAPI; diff --git a/packages/ws-worker/src/mock/lightning/index.ts b/packages/ws-worker/src/mock/lightning/index.ts deleted file mode 100644 index 5f3b83d4f..000000000 --- a/packages/ws-worker/src/mock/lightning/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -import createLightningServer from './server'; -export default createLightningServer; diff --git a/packages/ws-worker/src/mock/lightning/server.ts b/packages/ws-worker/src/mock/lightning/server.ts deleted file mode 100644 index 67f6aa8b3..000000000 --- a/packages/ws-worker/src/mock/lightning/server.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { EventEmitter } from 'node:events'; -import Koa from 'koa'; -import bodyParser from 'koa-bodyparser'; -import koaLogger from 'koa-logger'; -import createLogger, { - createMockLogger, - LogLevel, - Logger, -} from '@openfn/logger'; - -import createWebSocketAPI from './api-sockets'; -import createDevAPI from './api-dev'; -import { Attempt } from '../../types'; -import { ATTEMPT_LOG_PAYLOAD } from '../../events'; - -export type AttemptState = { - status: 'queued' | 'started' | 'complete'; - logs: ATTEMPT_LOG_PAYLOAD[]; -}; - -export type ServerState = { - queue: AttemptId[]; - - // list of credentials by id - credentials: Record; - - // list of attempts by id - attempts: Record; - - // list of dataclips by id - dataclips: Record; - - // Tracking state of known attempts - // TODO include the engine id and token - pending: Record; - - // Track all completed attempts here - results: Record; - - // event emitter for debugging and observability - events: EventEmitter; -}; - -export type LightningOptions = { - logger?: Logger; - logLevel?: LogLevel; - port?: string | number; -}; - -export type AttemptId = string; - -// a mock lightning server -const createLightningServer = (options: LightningOptions = {}) => { - const logger = options.logger || createMockLogger(); - - const state = { - credentials: {}, - attempts: {}, - dataclips: {}, - pending: {}, - - queue: [] as AttemptId[], - results: {}, - events: new EventEmitter(), - } as ServerState; - - const app = new Koa(); - app.use(bodyParser()); - - const port = options.port || 8888; - const server = app.listen(port); - logger.info('Listening on ', port); - - // Only create a http logger if there's a top-level logger passed - // This is a bit flaky really but whatever - if (options.logger) { - const httpLogger = createLogger('HTTP', { level: 'debug' }); - const klogger = koaLogger((str) => httpLogger.debug(str)); - app.use(klogger); - } - - // Setup the websocket API - const api = createWebSocketAPI( - state, - '/worker', // TODO I should option drive this - server, - options.logger && logger - ); - - app.use(createDevAPI(app as any, state, logger, api)); - - (app as any).destroy = () => { - server.close(); - }; - - return app; -}; - -export default createLightningServer; diff --git a/packages/ws-worker/src/mock/lightning/socket-server.ts b/packages/ws-worker/src/mock/lightning/socket-server.ts deleted file mode 100644 index 2655a2c9a..000000000 --- a/packages/ws-worker/src/mock/lightning/socket-server.ts +++ /dev/null @@ -1,268 +0,0 @@ -/** - * This module creates a mock pheonix socket server - * It uses a standard ws server but wraps messages up in a - * structure that pheonix sockets can understand - * It also adds some dev and debug APIs, useful for unit testing - */ -import { WebSocketServer, WebSocket } from 'ws'; -import querystring from 'query-string'; -// @ts-ignore -import { Serializer } from 'phoenix'; - -import { ATTEMPT_PREFIX, extractAttemptId } from './util'; -import { ServerState } from './server'; -import { stringify } from '../../util'; - -import type { Logger } from '@openfn/logger'; - -type Topic = string; - -const decoder = Serializer.decode.bind(Serializer); -const decode = (data: any) => new Promise((done) => decoder(data, done)); - -const encoder = Serializer.encode.bind(Serializer); -const encode = (data: any) => - new Promise((done) => { - if (data.payload?.response && data.payload.response instanceof Uint8Array) { - // special encoding logic if the payload is a buffer - // (we need to do this for dataclips) - data.payload.response = Array.from(data.payload.response); - } - encoder(data, done); - }); - -export type PhoenixEventStatus = 'ok' | 'error' | 'timeout'; - -// websocket with a couple of dev-friendly APIs -export type DevSocket = WebSocket & { - reply: (evt: PhoenixReply) => void; - sendJSON: ({ event, topic, ref }: PhoenixEvent) => void; -}; - -export type PhoenixEvent

= { - topic: Topic; - event: string; - payload: P; - ref: string; - join_ref: string; -}; - -export type PhoenixReply = { - topic: Topic; - payload: { - status: PhoenixEventStatus; - response?: R; - }; - ref: string; - join_ref: string; -}; - -type EventHandler = (ws: DevSocket, event: PhoenixEvent) => void; - -type CreateServerOptions = { - port?: number; - server: typeof WebSocketServer; - state: ServerState; - logger?: Logger; - onMessage?: (evt: PhoenixEvent) => void; -}; - -type MockSocketServer = typeof WebSocketServer & { - // Dev/debug APIs - listenToChannel: ( - topic: Topic, - fn: EventHandler - ) => { unsubscribe: () => void }; - waitForMessage: (topic: Topic, event: string) => Promise; - registerEvents: ( - topic: Topic, - events: Record - ) => { unsubscribe: () => void }; -}; - -function createServer({ - port = 8080, - server, - state, - logger, - onMessage = () => {}, -}: CreateServerOptions) { - const channels: Record> = { - // create a stub listener for pheonix to prevent errors - phoenix: new Set([() => null]), - }; - - const wsServer = - server || - new WebSocketServer({ - port, - }); - - if (!server) { - logger?.info('pheonix mock websocket server listening on', port); - } - - const events = { - // When joining a channel, we need to send a chan_reply_{ref} message back to the socket - phx_join: ( - ws: DevSocket, - { topic, ref, payload, join_ref }: PhoenixEvent - ) => { - let status: PhoenixEventStatus = 'ok'; - let response = 'ok'; - - // Validation on attempt:* channels - // TODO is this logic in the right place? - if (topic.startsWith(ATTEMPT_PREFIX)) { - const attemptId = extractAttemptId(topic); - if (!state.pending[attemptId]) { - status = 'error'; - response = 'invalid_attempt_id'; - } else if (!payload.token) { - // TODO better token validation here - status = 'error'; - response = 'invalid_token'; - } - } - ws.reply({ - topic, - payload: { status, response }, - ref, - join_ref, - }); - }, - }; - - // @ts-ignore something wierd about the wsServer typing - wsServer.on('connection', function (ws: DevSocket, req: any) { - logger?.info('new client connected'); - - // Ensure that a JWT token is added to the - const [_path, query] = req.url.split('?'); - const { token } = querystring.parse(query); - - // TODO for now, there's no validation on the token in this mock - - // If there is no token (or later, if invalid), close the connection immediately - if (!token) { - logger?.error('INVALID TOKEN'); - ws.close(); - - // TODO I'd love to send back a 403 here, not sure how to do it - // (and it's not important in the mock really) - return; - } - - ws.reply = async ({ - ref, - topic, - payload, - join_ref, - }: PhoenixReply) => { - // TODO only stringify payload if not a buffer - logger?.debug(`<< [${topic}] chan_reply_${ref} ` + stringify(payload)); - const evt = await encode({ - event: `chan_reply_${ref}`, - ref, - join_ref, - topic, - payload, - }); - // @ts-ignore - ws.send(evt); - }; - - ws.sendJSON = async ({ event, ref, topic, payload }: PhoenixEvent) => { - logger?.debug(`<< [${topic}] ${event} ` + stringify(payload)); - const evt = await encode({ - event, - ref, - topic, - payload: stringify(payload), // TODO do we stringify this? All of it? - }); - // @ts-ignore - ws.send(evt); - }; - - ws.on('message', async function (data: string) { - // decode the data - const evt = (await decode(data)) as PhoenixEvent; - onMessage(evt); - - if (evt.topic) { - // phx sends this info in each message - const { topic, event, payload, ref, join_ref } = evt; - - logger?.debug(`>> [${topic}] ${event} ${ref} :: ${stringify(payload)}`); - - if (event in events) { - // handle system/phoenix events - // @ts-ignore - events[event](ws, { topic, payload, ref, join_ref }); - } else { - // handle custom/user events - if (channels[topic] && channels[topic].size) { - channels[topic].forEach((fn) => { - fn(ws, { event, topic, payload, ref, join_ref }); - }); - } else { - // This behaviour is just a convenience for unit tesdting - ws.reply({ - ref, - join_ref, - topic, - payload: { - status: 'error', - response: `There are no listeners on channel ${topic}`, - }, - }); - } - } - } - }); - }); - - const mockServer = wsServer as MockSocketServer; - - // debug API - // TODO should this in fact be (topic, event, fn)? - mockServer.listenToChannel = (topic: Topic, fn: EventHandler) => { - if (!channels[topic]) { - channels[topic] = new Set(); - } - - channels[topic].add(fn); - - return { - unsubscribe: () => { - channels[topic].delete(fn); - }, - }; - }; - - mockServer.waitForMessage = (topic: Topic, event: string) => { - return new Promise((resolve) => { - const listener = mockServer.listenToChannel(topic, (_ws, e) => { - if (e.event === event) { - listener.unsubscribe(); - resolve(e); - } - }); - }); - }; - - mockServer.registerEvents = (topic: Topic, events) => { - // listen to all events in the channel - return mockServer.listenToChannel(topic, (ws, evt) => { - const { event } = evt; - // call the correct event handler for this event - if (events[event]) { - events[event](ws, evt); - } - }); - }; - - return mockServer as MockSocketServer; -} - -export default createServer; diff --git a/packages/ws-worker/src/mock/lightning/start.ts b/packages/ws-worker/src/mock/lightning/start.ts deleted file mode 100644 index 557d7b07f..000000000 --- a/packages/ws-worker/src/mock/lightning/start.ts +++ /dev/null @@ -1,29 +0,0 @@ -import yargs from 'yargs'; -import { hideBin } from 'yargs/helpers'; -import createLogger from '@openfn/logger'; - -import createLightningServer from './server'; - -type Args = { - _: string[]; - port?: number; -}; - -const args = yargs(hideBin(process.argv)) - .command('server', 'Start a mock lighting server') - .option('port', { - alias: 'p', - description: 'Port to run the server on', - type: 'number', - default: 8888, - }) - .parse() as Args; - -const logger = createLogger('LNG', { level: 'debug' }); - -createLightningServer({ - port: args.port, - logger, -}); - -logger.success('Started mock Lightning server on ', args.port); diff --git a/packages/ws-worker/src/mock/lightning/util.ts b/packages/ws-worker/src/mock/lightning/util.ts deleted file mode 100644 index 6fd5685e1..000000000 --- a/packages/ws-worker/src/mock/lightning/util.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const ATTEMPT_PREFIX = 'attempt:'; - -export const extractAttemptId = (topic: string) => - topic.substr(ATTEMPT_PREFIX.length); diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index b1176e1c1..2d8ebcc34 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -3,7 +3,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import createLogger, { LogLevel } from '@openfn/logger'; -// import createRTM from '@openfn/runtime-engine'; +import createRTE from '@openfn/engine-multi'; import createMockRTE from './mock/runtime-engine'; import createWorker from './server'; @@ -15,6 +15,7 @@ type Args = { secret?: string; loop?: boolean; log: LogLevel; + mock: boolean; }; const args = yargs(hideBin(process.argv)) @@ -51,6 +52,11 @@ const args = yargs(hideBin(process.argv)) default: true, type: 'boolean', }) + .option('mock', { + description: 'Use a mock runtime engine', + default: false, + type: 'boolean', + }) .parse() as Args; const logger = createLogger('SRV', { level: args.log }); @@ -66,19 +72,14 @@ if (args.lightning === 'mock') { args.secret = WORKER_SECRET; } - -// TODO the engine needs to take callbacks to load credential, and load state -// these in turn should utilise the websocket -// So either the server creates the runtime (which seems reasonable acutally?) -// Or the server calls a setCalbacks({ credential, state }) function on the engine -// Each of these takes the attemptId as the firsdt argument -// credential and state will lookup the right channel -// const engine = createEngine('rte', { repoDir: args.repoDir }); -// logger.debug('engine created'); - -// use the mock rtm for now -const engine = createMockRTE('rte'); -logger.debug('Mock RTE created'); +let engine; +if (args.mock) { + engine = createMockRTE('rte'); + logger.debug('Mock engine created'); +} else { + engine = createRTE('rte', { repoDir: args.repoDir }); + logger.debug('engine created'); +} createWorker(engine, { port: args.port, diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index ba0756c90..c3e3f5ef3 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -3,8 +3,8 @@ */ import test from 'ava'; +import createLightningServer from '@openfn/lightning-mock'; import createWorkerServer from '../src/server'; -import createLightningServer from '../src/mock/lightning'; import createMockRTE from '../src/mock/runtime-engine'; import * as e from '../src/events'; diff --git a/packages/ws-worker/test/mock/lightning.test.ts b/packages/ws-worker/test/mock/lightning.test.ts deleted file mode 100644 index 706174868..000000000 --- a/packages/ws-worker/test/mock/lightning.test.ts +++ /dev/null @@ -1,346 +0,0 @@ -import test from 'ava'; -import createLightningServer from '../../src/mock/lightning'; - -import { Socket } from 'phoenix'; -import { WebSocket } from 'ws'; - -import { attempts, credentials, dataclips } from './data'; -import { - ATTEMPT_COMPLETE, - ATTEMPT_COMPLETE_PAYLOAD, - ATTEMPT_LOG, - CLAIM, - GET_ATTEMPT, - GET_CREDENTIAL, - GET_DATACLIP, -} from '../../src/events'; -import type { Attempt, Channel } from '../../src/types'; -import { JSONLog } from '@openfn/logger'; - -const endpoint = 'ws://localhost:7777/worker'; - -const enc = new TextDecoder('utf-8'); - -let server; -let client; - -// Set up a lightning server and a phoenix socket client before each test -test.before( - () => - new Promise((done) => { - server = createLightningServer({ port: 7777 }); - - // Note that we need a token to connect, but the mock here - // doesn't (yet) do any validation on that token - client = new Socket(endpoint, { - params: { token: 'x.y.z' }, - timeout: 1000 * 120, - transport: WebSocket, - }); - client.onOpen(done); - client.connect(); - }) -); - -test.afterEach(() => { - server.reset(); -}); - -test.after(() => { - server.destroy(); -}); - -const attempt1 = attempts['attempt-1']; - -const join = (channelName: string, params: any = {}): Promise => - new Promise((done, reject) => { - const channel = client.channel(channelName, params); - channel - .join() - .receive('ok', () => { - done(channel); - }) - .receive('error', (err) => { - // err will be the response message on the payload (ie, invalid_token, invalid_attempt_id etc) - reject(new Error(err)); - }); - }); - -// Test some dev hooks -// enqueue attempt should add id to the queue and register the state, credentials and body -test.serial('should setup an attempt at /POST /attempt', async (t) => { - const state = server.getState(); - - t.is(Object.keys(state.credentials).length, 0); - t.is(Object.keys(state.attempts).length, 0); - t.is(Object.keys(state.attempts).length, 0); - - const attempt: Attempt = { - id: 'a', - triggers: [], - jobs: [ - { - body: 'abc', - dataclip: {}, // not sure how this will work on the attempt yet - credential: { - // this will be converted into a string for lazy loading - user: 'john', - password: 'rambo', - }, - }, - ], - edges: [], - }; - - await fetch('http://localhost:7777/attempt', { - method: 'POST', - body: JSON.stringify(attempt), - headers: { - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - }); - - const newState = server.getState(); - t.is(Object.keys(newState.attempts).length, 1); - const a = server.getAttempt('a'); - t.truthy(a); - t.is(server.getQueueLength(), 1); - - t.is(Object.keys(newState.credentials).length, 1); - - const job = a.jobs[0]; - t.assert(typeof job.credential === 'string'); - const c = server.getCredential(job.credential); - t.is(c.user, 'john'); -}); - -test.serial('provide a phoenix websocket at /worker', (t) => { - // client should be connected before this test runs - t.is(client.connectionState(), 'open'); -}); - -test.serial('reject ws connections without a token', (t) => { - return new Promise((done) => { - // client should be connected before this test runs - const socket = new Socket(endpoint, { transport: WebSocket }); - socket.onClose(() => { - t.pass(); - done(); - }); - socket.connect(); - }); -}); - -test.serial('respond to channel join requests', (t) => { - return new Promise(async (done, reject) => { - const channel = client.channel('x', {}); - - channel.join().receive('ok', (res) => { - t.is(res, 'ok'); - done(); - }); - }); -}); - -// TODO: only allow authorised workers to join workers -// TODO: only allow authorised attemtps to join an attempt channel - -test.serial( - 'claim attempt: reply for zero items if queue is empty', - (t) => - new Promise(async (done) => { - t.is(server.getQueueLength(), 0); - - const channel = await join('worker:queue'); - - // response is an array of attempt ids - channel.push(CLAIM).receive('ok', (response) => { - const { attempts } = response; - t.assert(Array.isArray(attempts)); - t.is(attempts.length, 0); - - t.is(server.getQueueLength(), 0); - done(); - }); - }) -); - -test.serial( - "claim attempt: reply with an attempt id if there's an attempt in the queue", - (t) => - new Promise(async (done) => { - server.enqueueAttempt(attempt1); - t.is(server.getQueueLength(), 1); - - const channel = await join('worker:queue'); - - // response is an array of attempt ids - channel.push(CLAIM).receive('ok', (response) => { - const { attempts } = response; - t.truthy(attempts); - t.is(attempts.length, 1); - t.deepEqual(attempts[0], { id: 'attempt-1', token: 'x.y.z' }); - - // ensure the server state has changed - t.is(server.getQueueLength(), 0); - done(); - }); - }) -); - -// TODO is it even worth doing this? Easier for a socket to pull one at a time? -// It would also ensure better distribution if 10 workers ask at the same time, they'll get -// one each then come back for more -test.todo('claim attempt: reply with multiple attempt ids'); - -test.serial('create a channel for an attempt', async (t) => { - server.startAttempt('wibble'); - await join('attempt:wibble', { token: 'a.b.c' }); - t.pass('connection ok'); -}); - -test.serial('do not allow to join a channel without a token', async (t) => { - server.startAttempt('wibble'); - await t.throwsAsync(() => join('attempt:wibble'), { - message: 'invalid_token', - }); -}); - -test.todo('do not allow to join a channel without a valid token'); - -test.serial('reject channels for attempts that are not started', async (t) => { - await t.throwsAsync(() => join('attempt:xyz'), { - message: 'invalid_attempt_id', - }); -}); - -test.serial('get attempt data through the attempt channel', async (t) => { - return new Promise(async (done) => { - server.registerAttempt(attempt1); - server.startAttempt(attempt1.id); - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(GET_ATTEMPT).receive('ok', (attempt) => { - t.deepEqual(attempt, attempt1); - done(); - }); - }); -}); - -test.serial.only( - 'complete an attempt through the attempt channel', - async (t) => { - return new Promise(async (done) => { - const a = attempt1; - server.registerAttempt(a); - server.startAttempt(a.id); - server.addDataclip('abc', { answer: 42 }); - - const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); - channel - .push(ATTEMPT_COMPLETE, { final_dataclip_id: 'abc' }) - .receive('ok', () => { - const { pending, results } = server.getState(); - t.deepEqual(pending[a.id], { status: 'complete', logs: [] }); - t.deepEqual(results[a.id].state, { answer: 42 }); - done(); - }); - }); - } -); - -test.serial('logs are saved and acknowledged', async (t) => { - return new Promise(async (done) => { - server.registerAttempt(attempt1); - server.startAttempt(attempt1.id); - - const log = { - attempt_id: attempt1.id, - level: 'info', - name: 'R/T', - message: ['Did the thing'], - time: new Date().getTime(), - } as JSONLog; - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_LOG, log).receive('ok', () => { - const { pending } = server.getState(); - const [savedLog] = pending[attempt1.id].logs; - t.deepEqual(savedLog, log); - done(); - }); - }); -}); - -test.serial('unsubscribe after attempt complete', async (t) => { - return new Promise(async (done) => { - const a = attempt1; - server.registerAttempt(a); - server.startAttempt(a.id); - - const channel = await join(`attempt:${a.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_COMPLETE).receive('ok', () => { - // After the complete event, the listener should unsubscribe to the channel - // The mock will send an error to any unhandled events in that channel - channel.push(ATTEMPT_COMPLETE).receive('error', () => { - t.pass(); - done(); - }); - }); - }); -}); - -test.serial('get credential through the attempt channel', async (t) => { - return new Promise(async (done) => { - server.startAttempt(attempt1.id); - server.addCredential('a', credentials['a']); - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(GET_CREDENTIAL, { id: 'a' }).receive('ok', (result) => { - t.deepEqual(result, credentials['a']); - done(); - }); - }); -}); - -test.serial('get dataclip through the attempt channel', async (t) => { - return new Promise(async (done) => { - server.startAttempt(attempt1.id); - server.addDataclip('d', dataclips['d']); - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(GET_DATACLIP, { id: 'd' }).receive('ok', (result) => { - const str = enc.decode(new Uint8Array(result)); - const dataclip = JSON.parse(str); - t.deepEqual(dataclip, dataclips['d']); - done(); - }); - }); -}); - -// TODO test that all events are proxied out to server.on - -test.serial( - 'waitForResult should return logs and dataclip when an attempt is completed', - async (t) => { - return new Promise(async (done) => { - const result = { answer: 42 }; - - server.startAttempt(attempt1.id); - server.addDataclip('result', result); - - server.waitForResult(attempt1.id).then((dataclip) => { - t.deepEqual(result, dataclip); - done(); - }); - - const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); - channel.push(ATTEMPT_COMPLETE, { - final_dataclip_id: 'result', - } as ATTEMPT_COMPLETE_PAYLOAD); - }); - } -); - -// test.serial('getLogs should return logs', async (t) => {}); diff --git a/packages/ws-worker/test/mock/socket-server.test.ts b/packages/ws-worker/test/mock/socket-server.test.ts deleted file mode 100644 index 2f7e5552d..000000000 --- a/packages/ws-worker/test/mock/socket-server.test.ts +++ /dev/null @@ -1,149 +0,0 @@ -import test from 'ava'; -import { Socket } from 'phoenix'; -import { WebSocket } from 'ws'; - -import createServer from '../../src/mock/lightning/socket-server'; - -let socket; -let server; -let messages; - -const wait = (duration = 10) => - new Promise((resolve) => { - setTimeout(resolve, duration); - }); - -test.beforeEach( - () => - new Promise((done) => { - messages = []; - // @ts-ignore I don't care about missing server options here - server = createServer({ onMessage: (evt) => messages.push(evt) }); - - socket = new Socket('ws://localhost:8080', { - transport: WebSocket, - params: { token: 'x.y.z' }, - }); - - socket.onOpen(done); - - socket.connect(); - }) -); - -test.afterEach(() => { - server.close(); -}); - -test.serial('respond to connection join requests', async (t) => { - return new Promise((resolve) => { - const channel = socket.channel('x', {}); - - channel - .join() - .receive('ok', (resp) => { - t.is(resp, 'ok'); - - channel.push('hello'); - resolve(); - }) - .receive('error', (e) => { - console.log(e); - }); - }); -}); - -test.serial('send a message', async (t) => { - return new Promise((resolve) => { - const channel = socket.channel('x', {}); - - server.listenToChannel('x', (_ws, { payload, event }) => { - t.is(event, 'hello'); - t.deepEqual(payload, { x: 1 }); - - resolve(); - }); - - channel.join(); - - channel.push('hello', { x: 1 }); - }); -}); - -test.serial('send a message only to one channel', async (t) => { - let didCallX = false; - let didCallY = false; - - const x = socket.channel('x', {}); - x.join(); - - const y = socket.channel('y', {}); - y.join(); - - server.listenToChannel('x', () => { - didCallX = true; - }); - server.listenToChannel('y', () => { - didCallY = true; - }); - - x.push('hello', { x: 1 }); - - await wait(); - - t.true(didCallX); - t.false(didCallY); -}); - -test.serial('unsubscribe', (t) => { - return new Promise(async (resolve) => { - let count = 0; - - const channel = socket.channel('x', {}); - channel.join(); - - const listener = server.listenToChannel('x', () => { - count++; - }); - - channel.push('hello', { x: 1 }); - await wait(100); - - t.is(count, 1); - - listener.unsubscribe(); - - channel.push('hello', { x: 1 }); - await wait(); - - t.is(count, 1); - - resolve(); - }); -}); - -test.serial('wait for message', async (t) => { - const channel = socket.channel('x', {}); - channel.join(); - - channel.push('hello', { x: 1 }); - - const result = await server.waitForMessage('x', 'hello'); - t.truthy(result); -}); - -test.serial('onMessage', (t) => { - return new Promise((done) => { - const channel = socket.channel('x', {}); - channel.join().receive('ok', async () => { - t.is(messages.length, 1); - t.is(messages[0].event, 'phx_join'); - - channel.push('hello', { x: 1 }); - await server.waitForMessage('x', 'hello'); - t.is(messages.length, 2); - t.is(messages[1].event, 'hello'); - done(); - }); - }); -}); diff --git a/packages/ws-worker/test/server.test.ts b/packages/ws-worker/test/server.test.ts deleted file mode 100644 index ced018d82..000000000 --- a/packages/ws-worker/test/server.test.ts +++ /dev/null @@ -1,68 +0,0 @@ -import test from 'ava'; -import WebSocket, { WebSocketServer } from 'ws'; - -import createServer from '../src/server'; -import connectToLightning from '../src/api/connect'; -import createMockRTE from '../src/mock/runtime-engine'; -import { sleep } from './util'; -import { mockChannel, mockSocket } from '../src/mock/sockets'; -import { CLAIM } from '../src/events'; - -// Unit tests against the worker server -// I don't think there will ever be much here because the server is mostly a pull - -let engine; -let server; -let cancel; - -const url = 'http://localhost:7777'; - -test.beforeEach(() => { - engine = createMockRTE(); -}); - -test.afterEach(() => { - cancel?.(); // cancel any workloops - server?.close(); // whatever -}); - -test.skip('healthcheck', async (t) => { - const server = createServer(engine, { port: 7777 }); - const result = await fetch(`${url}/healthcheck`); - t.is(result.status, 200); - const body = await result.text(); - t.is(body, 'OK'); -}); - -test.todo('do something if we fail to connect to lightning'); -test.todo("don't explode if no lightning endpoint is set (or maybe do?)"); - -// TODO this isn't testing anything now, see test/api/connect.test.ts -// Not a very thorough test -// test.only('connects to lightning', async (t) => { -// await connectToLightning('www', 'rtm', mockSocket); -// t.pass(); - -// // TODO connections to the same socket.channel should share listners, so I think I can test the channel -// }); - -// test('connects to websocket', (t) => { -// let didSayHello; - -// const wss = new WebSocketServer({ -// port: 8080, -// }); -// wss.on('message', () => { -// didSayHello = true; -// }); - -// rtm = createMockRTE(); -// server = createServer(rtm, { -// port: 7777, -// lightning: 'ws://localhost:8080', -// // TODO what if htere's some kind of onready hook? -// // TODO also we'll need some utility like waitForEvent -// }); - -// t.true(didSayHello); -// }); From c138b4bb7b2fdef93d712718744fd0a844508482 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:03:09 +0100 Subject: [PATCH 152/232] worker: fix close() --- packages/ws-worker/package.json | 2 +- packages/ws-worker/src/index.ts | 4 +++- packages/ws-worker/src/server.ts | 8 ++++++-- packages/ws-worker/test/integration.test.ts | 6 +++--- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 7ea4369db..a7959efba 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -2,7 +2,7 @@ "name": "@openfn/ws-worker", "version": "0.1.0", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", - "main": "index.js", + "main": "dist/index.js", "type": "module", "scripts": { "test": "pnpm ava --serial", diff --git a/packages/ws-worker/src/index.ts b/packages/ws-worker/src/index.ts index 1199af15d..e2d703ce5 100644 --- a/packages/ws-worker/src/index.ts +++ b/packages/ws-worker/src/index.ts @@ -1 +1,3 @@ -import './server'; +import createServer from './server'; + +export default createServer; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 32b885407..f0b16321f 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -38,12 +38,16 @@ function createServer(engine: any, options: ServerOptions = {}) { }) ); - app.listen(port); + const server = app.listen(port); logger.success('ws-worker listening on', port); + let killWorkloop; + (app as any).destroy = () => { // TODO close the work loop logger.info('Closing server'); + server.close(); + killWorkloop?.(); }; const router = new Router(); @@ -70,7 +74,7 @@ function createServer(engine: any, options: ServerOptions = {}) { if (!options.noLoop) { logger.info('Starting workloop'); // TODO maybe namespace the workloop logger differently? It's a bit annoying - startWorkloop(channel, startAttempt, logger, { + killWorkloop = startWorkloop(channel, startAttempt, logger, { maxBackoff: options.maxBackoff, // timeout: 1000 * 60, // TMP debug poll once per minute }); diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index c3e3f5ef3..48ae516ee 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -9,17 +9,17 @@ import createMockRTE from '../src/mock/runtime-engine'; import * as e from '../src/events'; let lng; -let engine; +let worker; const urls = { - engine: 'http://localhost:4567', + worker: 'http://localhost:4567', lng: 'ws://localhost:7654/worker', }; test.before(() => { // TODO give lightning the same secret and do some validation lng = createLightningServer({ port: 7654 }); - engine = createWorkerServer(createMockRTE('engine'), { + worker = createWorkerServer(createMockRTE('engine'), { port: 4567, lightning: urls.lng, secret: 'abc', From b5ad0c029dec69da5d44c92a77eeca25a6118773 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:03:54 +0100 Subject: [PATCH 153/232] lightning-mock: new events --- packages/lightning-mock/src/api-dev.ts | 2 +- packages/lightning-mock/src/api-sockets.ts | 4 ++++ packages/lightning-mock/src/events.ts | 7 +++++++ packages/lightning-mock/src/server.ts | 1 + packages/lightning-mock/src/socket-server.ts | 6 ++++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/lightning-mock/src/api-dev.ts b/packages/lightning-mock/src/api-dev.ts index 3af58a3dc..75d701e18 100644 --- a/packages/lightning-mock/src/api-dev.ts +++ b/packages/lightning-mock/src/api-dev.ts @@ -63,7 +63,7 @@ const setupDevAPI = ( app.getDataclip = (id: string) => state.dataclips[id]; - app.enqueueAttempt = (attempt: Attempt, workerId = 'rtm') => { + app.enqueueAttempt = (attempt: Attempt, workerId = 'rte') => { state.attempts[attempt.id] = attempt; state.results[attempt.id] = { workerId, // TODO diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 7c05aad07..06708ed9b 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -133,6 +133,10 @@ const createSocketAPI = ( return { startAttempt, + close: () => { + server.close(); + wss.close(); + }, }; // pull claim will try and pull a claim off the queue, diff --git a/packages/lightning-mock/src/events.ts b/packages/lightning-mock/src/events.ts index 4aca14ce4..f098dea44 100644 --- a/packages/lightning-mock/src/events.ts +++ b/packages/lightning-mock/src/events.ts @@ -5,6 +5,13 @@ import { Attempt } from './types'; * There is a danger of them diverging */ +// new client connected +export const CONNECT = 'socket:connect'; + +// client left or joined a channel +export const CHANNEL_JOIN = 'socket:channel-join'; +export const CHANNEL_LEAVE = 'socket:channel-leave'; + export const CLAIM = 'claim'; export type ClaimPayload = { demand?: number }; diff --git a/packages/lightning-mock/src/server.ts b/packages/lightning-mock/src/server.ts index 49dd217ca..c2fa36c4b 100644 --- a/packages/lightning-mock/src/server.ts +++ b/packages/lightning-mock/src/server.ts @@ -91,6 +91,7 @@ const createLightningServer = (options: LightningOptions = {}) => { (app as any).destroy = () => { server.close(); + api.close(); }; return app; diff --git a/packages/lightning-mock/src/socket-server.ts b/packages/lightning-mock/src/socket-server.ts index a34069637..cf874b618 100644 --- a/packages/lightning-mock/src/socket-server.ts +++ b/packages/lightning-mock/src/socket-server.ts @@ -14,6 +14,7 @@ import { ServerState } from './server'; import { stringify } from './util'; import type { Logger } from '@openfn/logger'; +import { CHANNEL_JOIN, CONNECT } from './events'; type Topic = string; @@ -124,6 +125,7 @@ function createServer({ response = 'invalid_token'; } } + state.events.emit(CHANNEL_JOIN, { channel: topic }); ws.reply({ topic, payload: { status, response }, @@ -136,6 +138,7 @@ function createServer({ // @ts-ignore something wierd about the wsServer typing wsServer.on('connection', function (ws: DevSocket, req: any) { logger?.info('new client connected'); + state.events.emit(CONNECT); // todo client details maybe // Ensure that a JWT token is added to the const [_path, query] = req.url.split('?'); @@ -187,6 +190,9 @@ function createServer({ ws.on('message', async function (data: string) { // decode the data const evt = (await decode(data)) as PhoenixEvent; + if (evt.event !== 'claim') { + console.log(evt); + } onMessage(evt); if (evt.topic) { From ef5883ddeaf38ef2822e27fe526ca8bf004b54cd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:31:50 +0100 Subject: [PATCH 154/232] engine-multi: ensure listen can be called before execute --- packages/engine-multi/src/engine.ts | 25 ++++++++-- packages/engine-multi/test/engine.test.ts | 61 +++++++++++------------ packages/ws-worker/src/api/execute.ts | 2 +- packages/ws-worker/src/api/workloop.ts | 1 - packages/ws-worker/src/server.ts | 2 +- 5 files changed, 54 insertions(+), 37 deletions(-) diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 71e684f6a..d143c90b4 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -95,6 +95,8 @@ export type EngineOptions = { const createEngine = (options: EngineOptions, workerPath?: string) => { const states: Record = {}; const contexts: Record = {}; + const deferredListeners: Record[]> = {}; + // TODO I think this is for later //const activeWorkflows: string[] = []; @@ -140,6 +142,7 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { // TODO are we totally sure this takes a standard xplan? // Well, it MUST have an ID or there's trouble const executeWrapper = (plan: ExecutionPlan) => { + options.logger!.debug('executing plan ', plan?.id ?? ''); const workflowId = plan.id!; // TODO throw if plan is invalid // Wait, don't throw because the server will die @@ -155,6 +158,12 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { contexts[workflowId] = createWorkflowEvents(engine, context, workflowId); + // Hook up any listeners passed to listen() that were called before execute + if (deferredListeners[workflowId]) { + deferredListeners[workflowId].forEach((l) => listen(workflowId, l)); + delete deferredListeners[workflowId]; + } + // TODO typing between the class and interface isn't right // @ts-ignore execute(context); @@ -176,11 +185,21 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { handlers: Record ) => { const events = contexts[workflowId]; - for (const evt in handlers) { - events.on(evt, handlers[evt]); + if (events) { + // If execute() was called, we'll have a context and we can subscribe directly + for (const evt in handlers) { + events.on(evt, handlers[evt]); + } + } else { + // if execute() wasn't called yet, cache the listeners and we'll hook them up later + if (!deferredListeners[workflowId]) { + deferredListeners[workflowId] = []; + } + deferredListeners[workflowId].push(handlers); } - // TODO return unsubscribe handle + // TODO return unsubscribe handle? + // How does this work if deferred? }; engine.emit('test'); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index a98c54fa2..dfd12bd4d 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -19,6 +19,9 @@ const options = { logger, repoDir: '.', // doesn't matter for the mock noCompile: true, // messy - needed to allow an expression to be passed as json + autoinstall: { + handleIsInstalled: async () => true, + }, }; test.afterEach(() => { @@ -72,17 +75,7 @@ test('use a custom worker path', (t) => { test('execute with test worker and trigger workflow-complete', (t) => { return new Promise((done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine( - { - logger, - repoDir: '.', - noCompile: true, - autoinstall: { - handleIsInstalled: async () => true, - }, - }, - p - ); + const engine = createEngine(options, p); const plan = { id: 'a', @@ -104,16 +97,7 @@ test('execute with test worker and trigger workflow-complete', (t) => { test('execute does not return internal state stuff', (t) => { return new Promise((done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine( - { - logger, - noCompile: true, - autoinstall: { - handleIsInstalled: async () => true, - }, - }, - p - ); + const engine = createEngine(options, p); const plan = { id: 'a', @@ -148,16 +132,7 @@ test('execute does not return internal state stuff', (t) => { test('listen to workflow-complete', (t) => { return new Promise((done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine( - { - logger, - noCompile: true, - autoinstall: { - handleIsInstalled: async () => true, - }, - }, - p - ); + const engine = createEngine(options, p); const plan = { id: 'a', @@ -178,3 +153,27 @@ test('listen to workflow-complete', (t) => { }); }); }); + +test('call listen before execute', (t) => { + return new Promise((done) => { + const p = path.resolve('src/test/worker-functions.js'); + const engine = createEngine(options, p); + + const plan = { + id: 'a', + jobs: [ + { + expression: '34', + }, + ], + }; + + engine.listen(plan.id, { + [e.WORKFLOW_COMPLETE]: ({ state }) => { + t.is(state, 34); + done(); + }, + }); + engine.execute(plan); + }); +}); diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index ba94af8cd..1e268e09b 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -62,7 +62,7 @@ export function execute( plan: ExecutionPlan ) { return new Promise(async (resolve) => { - logger.info('execute..'); + logger.info('execute...'); const state: AttemptState = { plan, diff --git a/packages/ws-worker/src/api/workloop.ts b/packages/ws-worker/src/api/workloop.ts index 0c182824f..112365f72 100644 --- a/packages/ws-worker/src/api/workloop.ts +++ b/packages/ws-worker/src/api/workloop.ts @@ -6,7 +6,6 @@ import type { Logger } from '@openfn/logger'; import claim from './claim'; -// TODO this needs to return some kind of cancel function const startWorkloop = ( channel: Channel, execute: (attempt: CLAIM_ATTEMPT) => void, diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index f0b16321f..74baa1743 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -41,7 +41,7 @@ function createServer(engine: any, options: ServerOptions = {}) { const server = app.listen(port); logger.success('ws-worker listening on', port); - let killWorkloop; + let killWorkloop: () => void; (app as any).destroy = () => { // TODO close the work loop From 697533a12dfa236198d1c55b463a1c38324d882b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:42:43 +0100 Subject: [PATCH 155/232] worker-tests: added some basic integration tests --- integration-tests/worker/package.json | 37 +++ integration-tests/worker/readme.md | 1 + .../worker/test/integration.test.ts | 81 +++++++ integration-tests/worker/tsconfig.json | 14 ++ pnpm-lock.yaml | 224 ++++++++++++++---- 5 files changed, 314 insertions(+), 43 deletions(-) create mode 100644 integration-tests/worker/package.json create mode 100644 integration-tests/worker/readme.md create mode 100644 integration-tests/worker/test/integration.test.ts create mode 100644 integration-tests/worker/tsconfig.json diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json new file mode 100644 index 000000000..bb6d9d1e4 --- /dev/null +++ b/integration-tests/worker/package.json @@ -0,0 +1,37 @@ +{ + "name": "@openfn/integration-tests-worker", + "private": true, + "version": "1.0.0", + "description": "Lightning WOrker integration tests", + "author": "Open Function Group ", + "license": "ISC", + "type": "module", + "scripts": { + "clean": "rimraf dist repo", + "build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/worker/dist --no-version", + "build": "pnpm build:pack && docker build --tag worker-integration-tests .", + "start": "docker run worker-integration-tests", + "test": "npx ava -s --timeout 2m && pnpm clean", + "test:dev": "pnpm ava -s && pnpm clean" + }, + "dependencies": { + "@openfn/engine-multi": "workspace:^", + "@openfn/lightning-mock": "workspace:^", + "@openfn/logger": "workspace:^", + "@openfn/ws-worker": "workspace:^", + "@types/node": "^18.15.13", + "ava": "5.3.1", + "date-fns": "^2.30.0", + "rimraf": "^3.0.2", + "ts-node": "10.8.1", + "tslib": "^2.4.0", + "typescript": "^5.1.6" + }, + "files": [ + "dist", + "README.md" + ], + "devDependencies": { + "@types/rimraf": "^3.0.2" + } +} diff --git a/integration-tests/worker/readme.md b/integration-tests/worker/readme.md new file mode 100644 index 000000000..4dd920308 --- /dev/null +++ b/integration-tests/worker/readme.md @@ -0,0 +1 @@ +Integration tests for Lightning Worker architecture. diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts new file mode 100644 index 000000000..6e9935265 --- /dev/null +++ b/integration-tests/worker/test/integration.test.ts @@ -0,0 +1,81 @@ +import test from 'ava'; + +import createLightningServer from '@openfn/lightning-mock'; + +import createEngine from '@openfn/engine-multi'; +import createWorkerServer from '@openfn/ws-worker'; + +import createLogger, { createMockLogger } from '@openfn/logger'; + +let lightning; +let worker; + +test.afterEach(() => { + lightning.destroy(); + worker.destroy(); +}); + +const initLightning = () => { + lightning = createLightningServer({ port: 9999 }); +}; + +const initWorker = () => { + const engine = createEngine({ + // logger: createLogger('engine', { level: 'debug' }), + logger: createMockLogger(), + runtimeLogger: createMockLogger(), + }); + + worker = createWorkerServer(engine, { + logger: createMockLogger(), + // logger: createLogger('worker', { level: 'debug' }), + port: 2222, + lightning: 'ws://localhost:9999/worker', + secret: 'abc', // TODO use a more realistic secret + }); +}; + +test('should connect to lightning', (t) => { + return new Promise((done) => { + initLightning(); + lightning.on('socket:connect', () => { + t.pass('connection recieved'); + done(); + }); + initWorker(); + }); +}); + +test('should join attempts queue channel', (t) => { + return new Promise((done) => { + initLightning(); + lightning.on('socket:channel-join', ({ channel }) => { + if (channel === 'worker:queue') { + t.pass('joined channel'); + done(); + } + }); + initWorker(); + }); +}); + +test.only('should run a simple job with no compilation', (t) => { + return new Promise((done) => { + initLightning(); + lightning.on('attempt:complete', (evt) => { + t.pass('completed attempt'); + done(); + }); + initWorker(); + + lightning.enqueueAttempt({ + id: 'a1', + jobs: [ + { + id: 'j1', + body: 'const fn = (f) => (state) => f(state); fn(() => ({ data: { answer: 42} }))', + }, + ], + }); + }); +}); diff --git a/integration-tests/worker/tsconfig.json b/integration-tests/worker/tsconfig.json new file mode 100644 index 000000000..9bffa80cb --- /dev/null +++ b/integration-tests/worker/tsconfig.json @@ -0,0 +1,14 @@ +{ + "ts-node": { + "experimentalSpecifierResolution": "node" + }, + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "module": "es2020", + "moduleResolution": "node", + "allowJs": true, + "isolatedModules": true, + "noEmit": true, + "skipLibCheck": true + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4ac183c39..d9c3750ff 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -110,6 +110,46 @@ importers: specifier: npm:@openfn/language-common@^1.11.0 version: /@openfn/language-common@1.11.0 + integration-tests/worker: + dependencies: + '@openfn/engine-multi': + specifier: workspace:^ + version: link:../../packages/engine-multi + '@openfn/lightning-mock': + specifier: workspace:^ + version: link:../../packages/lightning-mock + '@openfn/logger': + specifier: workspace:^ + version: link:../../packages/logger + '@openfn/ws-worker': + specifier: workspace:^ + version: link:../../packages/ws-worker + '@types/node': + specifier: ^18.15.13 + version: 18.15.13 + ava: + specifier: 5.3.1 + version: 5.3.1 + date-fns: + specifier: ^2.30.0 + version: 2.30.0 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + ts-node: + specifier: 10.8.1 + version: 10.8.1(@types/node@18.15.13)(typescript@5.1.6) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + typescript: + specifier: ^5.1.6 + version: 5.1.6 + devDependencies: + '@types/rimraf': + specifier: ^3.0.2 + version: 3.0.2 + packages/cli: dependencies: '@inquirer/prompts': @@ -351,8 +391,8 @@ importers: specifier: ^2.13.4 version: 2.13.4 workerpool: - specifier: ^6.2.1 - version: 6.2.1 + specifier: ^6.5.1 + version: 6.5.1 devDependencies: '@types/koa': specifier: ^2.13.5 @@ -364,8 +404,8 @@ importers: specifier: ^1.19.2 version: 1.19.2 '@types/workerpool': - specifier: ^6.1.0 - version: 6.1.0 + specifier: ^6.4.4 + version: 6.4.4 ava: specifier: 5.3.1 version: 5.3.1 @@ -388,6 +428,103 @@ importers: specifier: ^5.1.6 version: 5.1.6 + packages/lightning-mock: + dependencies: + '@koa/router': + specifier: ^12.0.0 + version: 12.0.0 + '@openfn/engine-multi': + specifier: workspace:* + version: link:../engine-multi + '@openfn/logger': + specifier: workspace:* + version: link:../logger + '@openfn/runtime': + specifier: workspace:* + version: link:../runtime + '@types/koa-logger': + specifier: ^3.1.2 + version: 3.1.2 + '@types/ws': + specifier: ^8.5.6 + version: 8.5.6 + fast-safe-stringify: + specifier: ^2.1.1 + version: 2.1.1 + jose: + specifier: ^4.14.6 + version: 4.14.6 + koa: + specifier: ^2.13.4 + version: 2.13.4 + koa-bodyparser: + specifier: ^4.4.0 + version: 4.4.0 + koa-logger: + specifier: ^3.2.1 + version: 3.2.1 + phoenix: + specifier: ^1.7.7 + version: 1.7.7 + ws: + specifier: ^8.14.1 + version: 8.14.1 + devDependencies: + '@types/koa': + specifier: ^2.13.5 + version: 2.13.5 + '@types/koa-bodyparser': + specifier: ^4.3.10 + version: 4.3.10 + '@types/koa-route': + specifier: ^3.2.6 + version: 3.2.6 + '@types/koa-websocket': + specifier: ^5.0.8 + version: 5.0.8 + '@types/koa__router': + specifier: ^12.0.1 + version: 12.0.1 + '@types/node': + specifier: ^18.15.3 + version: 18.15.13 + '@types/nodemon': + specifier: 1.19.3 + version: 1.19.3 + '@types/phoenix': + specifier: ^1.6.2 + version: 1.6.2 + '@types/yargs': + specifier: ^17.0.12 + version: 17.0.24 + ava: + specifier: 5.1.0 + version: 5.1.0 + koa-route: + specifier: ^3.2.0 + version: 3.2.0 + koa-websocket: + specifier: ^7.0.0 + version: 7.0.0 + query-string: + specifier: ^8.1.0 + version: 8.1.0 + ts-node: + specifier: ^10.9.1 + version: 10.9.1(@types/node@18.15.13)(typescript@4.6.4) + tslib: + specifier: ^2.4.0 + version: 2.4.0 + tsup: + specifier: ^6.2.3 + version: 6.2.3(ts-node@10.9.1)(typescript@4.6.4) + typescript: + specifier: ^4.6.4 + version: 4.6.4 + yargs: + specifier: ^17.6.2 + version: 17.7.2 + packages/logger: dependencies: '@inquirer/confirm': @@ -507,6 +644,9 @@ importers: specifier: ^8.14.1 version: 8.14.1 devDependencies: + '@openfn/lightning-mock': + specifier: workspace:* + version: link:../lightning-mock '@types/koa': specifier: ^2.13.5 version: 2.13.5 @@ -1214,6 +1354,11 @@ packages: heap: 0.2.7 dev: false + /@fastify/busboy@2.0.0: + resolution: {integrity: sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==} + engines: {node: '>=14'} + dev: false + /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1484,12 +1629,12 @@ packages: dependencies: ajv: 8.12.0 axios: 1.1.3 - csv-parse: 5.4.0 + csv-parse: 5.5.2 csvtojson: 2.0.10 date-fns: 2.30.0 jsonpath-plus: 4.0.0 lodash: 4.17.21 - undici: 5.24.0 + undici: 5.26.3 transitivePeerDependencies: - debug dev: false @@ -1817,7 +1962,7 @@ packages: resolution: {integrity: sha512-F3OznnSLAUxFrCEu/L5PY8+ny8DtcFRjx7fZZ9bycvXRi3KPTRS9HOitGZwvPg0juRhXFWIeKX58cnX5YqLohQ==} dependencies: '@types/glob': 8.1.0 - '@types/node': 18.15.3 + '@types/node': 18.15.13 dev: true /@types/scheduler@0.16.2: @@ -1844,8 +1989,8 @@ packages: resolution: {integrity: sha512-ONpcZAEYlbPx4EtJwfTyCDQJGUpKf4sEcuySdCVjK5Fj/3vHp5HII1fqa1/+qrsLnpYELCQTfVW/awsGJePoIg==} dev: true - /@types/workerpool@6.1.0: - resolution: {integrity: sha512-C+J/c1BHyc351xJuiH2Jbe+V9hjf5mCzRP0UK4KEpF5SpuU+vJ/FC5GLZsCU/PJpp/3I6Uwtfm3DG7Lmrb7LOQ==} + /@types/workerpool@6.4.4: + resolution: {integrity: sha512-rpYFug3QyKzQ7+y/x8BCTEseMorTyr9DiY3ao5KxzWJPtFyx/HL0SSLtJlRjUSpBeaMd/zn7hnLaWOb8WRFnnQ==} dependencies: '@types/node': 18.15.13 dev: true @@ -1929,11 +2074,6 @@ packages: hasBin: true dev: false - /acorn@8.8.1: - resolution: {integrity: sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==} - engines: {node: '>=0.4.0'} - hasBin: true - /agent-base@6.0.2: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} @@ -2280,7 +2420,7 @@ packages: /axios@1.1.3: resolution: {integrity: sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA==} dependencies: - follow-redirects: 1.15.2 + follow-redirects: 1.15.3 form-data: 4.0.0 proxy-from-env: 1.1.0 transitivePeerDependencies: @@ -2464,13 +2604,6 @@ packages: load-tsconfig: 0.2.5 dev: true - /busboy@1.6.0: - resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} - engines: {node: '>=10.16.0'} - dependencies: - streamsearch: 1.1.0 - dev: false - /bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -2629,7 +2762,7 @@ packages: normalize-path: 3.0.0 readdirp: 3.6.0 optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 /chownr@2.0.0: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} @@ -2914,8 +3047,8 @@ packages: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} dev: true - /csv-parse@5.4.0: - resolution: {integrity: sha512-JiQosUWiOFgp4hQn0an+SBoV9IKdqzhROM0iiN4LB7UpfJBlsSJlWl9nq4zGgxgMAzHJ6V4t29VAVD+3+2NJAg==} + /csv-parse@5.5.2: + resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} dev: false /csv-stringify@5.6.5: @@ -4057,6 +4190,16 @@ packages: peerDependenciesMeta: debug: optional: true + dev: true + + /follow-redirects@1.15.3: + resolution: {integrity: sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4146,12 +4289,12 @@ packages: requiresBuild: true dependencies: bindings: 1.5.0 - nan: 2.17.0 + nan: 2.18.0 dev: true optional: true - /fsevents@2.3.2: - resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] requiresBuild: true @@ -5576,8 +5719,8 @@ packages: thenify-all: 1.6.0 dev: true - /nan@2.17.0: - resolution: {integrity: sha512-2ZTgtl0nJsO0KQCjEpxcIr5D+Yv90plTitZt9JBfQvVJDS5seMl3FOvsh3+9CoYWXf/1l5OaZzzF6nDm4cagaQ==} + /nan@2.18.0: + resolution: {integrity: sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==} requiresBuild: true dev: true optional: true @@ -6616,7 +6759,7 @@ packages: engines: {node: '>=10.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /rollup@3.27.2: @@ -6624,7 +6767,7 @@ packages: engines: {node: '>=14.18.0', npm: '>=8.0.0'} hasBin: true optionalDependencies: - fsevents: 2.3.2 + fsevents: 2.3.3 dev: true /run-async@3.0.0: @@ -7023,11 +7166,6 @@ packages: mixme: 0.5.4 dev: true - /streamsearch@1.1.0: - resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} - engines: {node: '>=10.0.0'} - dev: false - /streamx@2.13.0: resolution: {integrity: sha512-9jD4uoX0juNSIcv4PazT+97FpM4Mww3cp7PM23HRTLANhgb7K7n1mB45guH/kT5F4enl04kApOM3EeoUXSPfvw==} dependencies: @@ -7357,7 +7495,7 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 '@types/node': 18.15.13 - acorn: 8.8.1 + acorn: 8.10.0 acorn-walk: 8.2.0 arg: 4.1.3 create-require: 1.1.1 @@ -7645,11 +7783,11 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true - /undici@5.24.0: - resolution: {integrity: sha512-OKlckxBjFl0oXxcj9FU6oB8fDAaiRUq+D8jrFWGmOfI/gIyjk/IeS75LMzgYKUaeHzLUcYvf9bbJGSrUwTfwwQ==} + /undici@5.26.3: + resolution: {integrity: sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==} engines: {node: '>=14.0'} dependencies: - busboy: 1.6.0 + '@fastify/busboy': 2.0.0 dev: false /union-value@1.0.1: @@ -7852,8 +7990,8 @@ packages: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} dev: false - /workerpool@6.2.1: - resolution: {integrity: sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==} + /workerpool@6.5.1: + resolution: {integrity: sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==} dev: false /wrap-ansi@6.2.0: From 8d1683ee57ccbde719e0918d264a67013e19a4bd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:43:15 +0100 Subject: [PATCH 156/232] engine: tidying --- packages/engine-multi/src/api/lifecycle.ts | 2 ++ packages/engine-multi/src/engine.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 151235ed5..e4f27ff7f 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -83,6 +83,8 @@ export const log = ( // // I'm sure there are nicer, more elegant ways of doing this // message: [`[${workflowId}]`, ...message.message], // }; + // TODO: if these are logs from within the runtime, + // should we use context.runtimeLogger ? context.logger.proxy(event.message); context.emit(externalEvents.WORKFLOW_LOG, { diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index d143c90b4..5fb0dc76b 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -80,6 +80,7 @@ class ExecutionContext extends EventEmitter { export type EngineOptions = { repoDir: string; logger: Logger; + runtimelogger?: Logger; resolvers?: LazyResolvers; From 224f8d98ee70e661561cc57915a7d311a7f52c00 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:43:45 +0100 Subject: [PATCH 157/232] lightning-mock: remove logging --- packages/lightning-mock/src/socket-server.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/lightning-mock/src/socket-server.ts b/packages/lightning-mock/src/socket-server.ts index cf874b618..1644603ed 100644 --- a/packages/lightning-mock/src/socket-server.ts +++ b/packages/lightning-mock/src/socket-server.ts @@ -190,9 +190,9 @@ function createServer({ ws.on('message', async function (data: string) { // decode the data const evt = (await decode(data)) as PhoenixEvent; - if (evt.event !== 'claim') { - console.log(evt); - } + // if (evt.event !== 'claim') { + // console.log(evt); + // } onMessage(evt); if (evt.topic) { From 84d3d209c76ba6a5e34596f33aeecc58f799dbf0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 15:44:09 +0100 Subject: [PATCH 158/232] cli-tests: update package name --- integration-tests/cli/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/cli/package.json b/integration-tests/cli/package.json index 9c9deea44..d73270380 100644 --- a/integration-tests/cli/package.json +++ b/integration-tests/cli/package.json @@ -1,7 +1,7 @@ { - "name": "@openfn/integration-tests", + "name": "@openfn/integration-tests-cli", "private": true, - "version": "0.0.1", + "version": "1.0.0", "description": "CLI integration tests", "author": "Open Function Group ", "license": "ISC", From 4db3042f1c8c40cee0a9868e1144fc641675a9ca Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 17:11:10 +0100 Subject: [PATCH 159/232] engine: emit event after autoinstall --- packages/engine-multi/src/api/autoinstall.ts | 14 ++++- packages/engine-multi/src/engine.ts | 2 +- packages/engine-multi/src/events.ts | 32 +++++----- .../engine-multi/test/api/autoinstall.test.ts | 61 ++++++++++++++----- 4 files changed, 74 insertions(+), 35 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 1aa8eab0d..569559bf0 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -11,6 +11,7 @@ import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import type { ExecutionContext } from '../types'; +import { AUTOINSTALL_COMPLETE } from '../events'; // none of these options should be on the plan actually export type AutoinstallOptions = { @@ -57,16 +58,27 @@ const autoinstall = async (context: ExecutionContext): Promise => { // Return a path name to this module for the linker to use later // TODO this is all a bit rushed const alias = getAliasedName(a); - const { name } = getNameAndVersion(a); + const { name, version } = getNameAndVersion(a); paths[name] = { path: `${repoDir}/node_modules/${alias}` }; const needsInstalling = !(await isInstalledFn(a, repoDir, logger)); if (needsInstalling) { if (!pending[a]) { // add a promise to the pending array + const startTime = Date.now(); pending[a] = installFn(a, repoDir, logger).then(() => { + const duration = Date.now() - startTime; + + logger.success(`autoinstalled ${a} in ${duration / 1000}s`); + context.emit(AUTOINSTALL_COMPLETE, { + module: name, + version: version!, + duration, + }); delete pending[a]; }); + // TODO catch, log and emit + // This should trigger a crash state } // Return the pending promise (safe to do this multiple times) await pending[a].then(); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 5fb0dc76b..21524cb49 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -56,7 +56,7 @@ class Engine extends EventEmitter {} // TODO this is actually the api that each execution gets // its nice to separate that from the engine a bit -class ExecutionContext extends EventEmitter { +export class ExecutionContext extends EventEmitter { state: WorkflowState; logger: Logger; callWorker: CallWorker; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index d67b25e72..171259924 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -20,6 +20,10 @@ export const WORKFLOW_LOG = 'workflow-log'; export const WORKFLOW_EDGE_RESOLVED = 'workflow-edge-resolved'; +export const AUTOINSTALL_COMPLETE = 'autoinstall-complete'; + +export const AUTOINSTALL_ERROR = 'autoinstall-error'; + export type EventMap = { [WORKFLOW_START]: WorkflowStartPayload; [WORKFLOW_COMPLETE]: WorkflowCompletePayload; @@ -27,6 +31,7 @@ export type EventMap = { [JOB_COMPLETE]: JobCompletePayload; [WORKFLOW_LOG]: WorkerLogPayload; [WORKFLOW_ERROR]: WorkflowErrorPayload; + [AUTOINSTALL_COMPLETE]: AutoinstallCompletePayload; }; export type ExternalEvents = keyof EventMap; @@ -36,46 +41,37 @@ interface ExternalEvent { workflowId: string; } -export interface WorkflowStartPayload extends ExternalEvent { - threadId: string; - workflowId: string; -} +export interface WorkflowStartPayload extends ExternalEvent {} export interface WorkflowCompletePayload extends ExternalEvent { - threadId: string; - workflowId: string; state: any; duration: number; } export interface WorkflowErrorPayload extends ExternalEvent { - threadId: string; - workflowId: string; type: string; message: string; } export interface JobStartPayload extends ExternalEvent { - threadId: string; - workflowId: string; jobId: string; } export interface JobCompletePayload extends ExternalEvent { - threadId: string; - workflowId: string; jobId: string; state: any; // the result state } -export interface WorkerLogPayload extends ExternalEvent, JSONLog { - threadId: string; - workflowId: string; -} +export interface WorkerLogPayload extends ExternalEvent, JSONLog {} export interface EdgeResolvedPayload extends ExternalEvent { - threadId: string; - workflowId: string; edgeId: string; // interesting, we don't really have this yet. Is index more appropriate? key? yeah, it's target node basically result: boolean; } + +// workflow id doesn't really help here? +export interface AutoinstallCompletePayload { + module: string; + version: string; + duration: number; +} diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index f51f0c64f..6c395a793 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -2,7 +2,8 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; -import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; +import { AUTOINSTALL_COMPLETE } from '../../src/events'; +import { ExecutionContext } from '../../src/engine'; type PackageJson = { name: string; @@ -20,21 +21,25 @@ const mockHandleInstall = async (specifier: string): Promise => const logger = createMockLogger(); const createContext = (autoinstallOpts?, jobs?: any[]) => - ({ - logger, + new ExecutionContext({ state: { + id: 'x', + status: 'pending', + options: {}, plan: { jobs: jobs || [{ adaptor: 'x@1.0.0' }], }, }, + logger, + callWorker: () => {}, options: { - repoDir: '.', + repoDir: 'tmp/repo', autoinstall: autoinstallOpts || { handleInstall: mockHandleInstall, handleIsInstalled: mockIsInstalled, }, }, - } as unknown as ExecutionContext); + }); test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ @@ -146,20 +151,46 @@ test.serial('autoinstall: return a map to modules', async (t) => { }, ]; - const context = createContext(null, jobs); - context.options = { - repoDir: 'a/b/c', - autoinstall: { - skipRepoValidation: true, - handleInstall: async () => {}, - handleIsInstalled: async () => false, - }, + const autoinstallOpts = { + skipRepoValidation: true, + handleInstall: async () => {}, + handleIsInstalled: async () => false, }; + const context = createContext(autoinstallOpts, jobs); const result = await autoinstall(context); t.deepEqual(result, { - common: { path: 'a/b/c/node_modules/common_1.0.0' }, - http: { path: 'a/b/c/node_modules/http_1.0.0' }, + common: { path: 'tmp/repo/node_modules/common_1.0.0' }, + http: { path: 'tmp/repo/node_modules/http_1.0.0' }, }); }); + +test.serial('autoinstall: emit an event on completion', async (t) => { + let event; + const jobs = [ + { + adaptor: 'common@1.0.0', + }, + ]; + + const autoinstallOpts = { + skipRepoValidation: true, + handleInstall: async () => new Promise((done) => setTimeout(done, 50)), + handleIsInstalled: async () => false, + }; + const context = createContext(autoinstallOpts, jobs); + + context.on(AUTOINSTALL_COMPLETE, (evt) => { + event = evt; + }); + + await autoinstall(context); + + t.truthy(event); + t.is(event.module, 'common'); + t.is(event.version, '1.0.0'); + t.assert(event.duration >= 50); +}); + +test.todo('autoinstall: emit on error'); From fdf478597c58e518d93a845ed5a14deb6cc34fb4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 18:17:59 +0100 Subject: [PATCH 160/232] worker: fix events for autoinstall Also an emitted event no longer has to include workflowId --- packages/engine-multi/src/api.ts | 4 +- packages/engine-multi/src/api/autoinstall.ts | 38 ++++++++++++------- packages/engine-multi/src/api/execute.ts | 1 - packages/engine-multi/src/api/lifecycle.ts | 6 +-- packages/engine-multi/src/engine.ts | 7 ++++ packages/engine-multi/src/events.ts | 13 +++++-- packages/engine-multi/src/types.d.ts | 5 ++- .../engine-multi/test/api/execute.test.ts | 18 ++++----- .../engine-multi/test/api/lifecycle.test.ts | 34 ++++++++--------- 9 files changed, 75 insertions(+), 51 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 0e4130d19..bf6408efc 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -76,7 +76,9 @@ const createAPI = function (options: RTEOptions = {}) { execute: engine.execute, listen: engine.listen, - // TODO what about a general on or once? + // expose a hook to listen to internal events + // @ts-ignore + // on: (...args) => engine.on(...args), }; }; diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 569559bf0..bb02806d4 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -11,7 +11,7 @@ import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import type { ExecutionContext } from '../types'; -import { AUTOINSTALL_COMPLETE } from '../events'; +import { AUTOINSTALL_COMPLETE, AUTOINSTALL_ERROR } from '../events'; // none of these options should be on the plan actually export type AutoinstallOptions = { @@ -64,23 +64,35 @@ const autoinstall = async (context: ExecutionContext): Promise => { const needsInstalling = !(await isInstalledFn(a, repoDir, logger)); if (needsInstalling) { if (!pending[a]) { + // TODO because autoinstall can take a while, we should emit that we're starting // add a promise to the pending array const startTime = Date.now(); - pending[a] = installFn(a, repoDir, logger).then(() => { - const duration = Date.now() - startTime; - - logger.success(`autoinstalled ${a} in ${duration / 1000}s`); - context.emit(AUTOINSTALL_COMPLETE, { - module: name, - version: version!, - duration, + pending[a] = installFn(a, repoDir, logger) + .then(() => { + const duration = Date.now() - startTime; + + logger.success(`autoinstalled ${a} in ${duration / 1000}s`); + context.emit(AUTOINSTALL_COMPLETE, { + module: name, + version: version!, + duration, + }); + delete pending[a]; + }) + .catch((e) => { + const duration = Date.now() - startTime; + console.log('AUTOINSTALL ERROR'); + console.log(e); + context.emit(AUTOINSTALL_ERROR, { + module: name, + version: version!, + duration, + message: e.message || e.toString(), + }); }); - delete pending[a]; - }); - // TODO catch, log and emit - // This should trigger a crash state } // Return the pending promise (safe to do this multiple times) + // TODO if this is a chained promise, emit something like "using cache for ${name}" await pending[a].then(); } } diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 2ced96a48..3cbee1571 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -11,7 +11,6 @@ const execute = async (context: ExecutionContext) => { const adaptorPaths = await autoinstall(context); await compile(context); - const events = { [workerEvents.WORKFLOW_START]: (evt: workerEvents.WorkflowStartEvent) => { workflowStart(context, evt); diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index e4f27ff7f..a2adc972d 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -34,8 +34,6 @@ export const workflowStart = ( // forward the event on to any external listeners context.emit(externalEvents.WORKFLOW_START, { threadId, - workflowId, // if this is a bespoke emitter it can be implied, which is nice - // Should we publish anything else here? }); }; @@ -64,7 +62,6 @@ export const workflowComplete = ( // forward the event on to any external listeners context.emit(externalEvents.WORKFLOW_COMPLETE, { - workflowId, threadId, duration: state.duration, state: result, @@ -75,7 +72,7 @@ export const log = ( context: ExecutionContext, event: internalEvents.LogEvent ) => { - const { workflowId, threadId } = event; + const { threadId } = event; // // TODO not sure about this stuff, I think we can drop it? // const newMessage = { // ...message, @@ -88,7 +85,6 @@ export const log = ( context.logger.proxy(event.message); context.emit(externalEvents.WORKFLOW_LOG, { - workflowId, threadId, ...event.message, }); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 21524cb49..d97785998 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -74,6 +74,13 @@ export class ExecutionContext extends EventEmitter { this.state = state; this.options = options; } + + // override emit to add the workflowId to all events + // @ts-ignore + emit(event, payload) { + payload.workflowId = this.state.id; + super.emit(event, payload); + } } // The engine is way more strict about options diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 171259924..53ce9dd7d 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -32,12 +32,13 @@ export type EventMap = { [WORKFLOW_LOG]: WorkerLogPayload; [WORKFLOW_ERROR]: WorkflowErrorPayload; [AUTOINSTALL_COMPLETE]: AutoinstallCompletePayload; + [AUTOINSTALL_ERROR]: AutoinstallErrorPayload; }; export type ExternalEvents = keyof EventMap; interface ExternalEvent { - threadId: string; + threadId?: string; workflowId: string; } @@ -69,9 +70,15 @@ export interface EdgeResolvedPayload extends ExternalEvent { result: boolean; } -// workflow id doesn't really help here? -export interface AutoinstallCompletePayload { +export interface AutoinstallCompletePayload extends ExternalEvent { module: string; version: string; duration: number; } + +export interface AutoinstallErrorPayload extends ExternalEvent { + module: string; + version: string; + duration: number; + message: string; +} diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.d.ts index 643bd3db7..ec611e031 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.d.ts @@ -57,7 +57,10 @@ export interface ExecutionContext extends EventEmitter { logger: Logger; callWorker: CallWorker; - emit(event: T, payload: EventMap[T]): boolean; + emit( + event: T, + payload: Omit + ): boolean; } export interface EngineAPI extends EventEmitter { diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index f01f89b7a..d30e74247 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -1,7 +1,7 @@ import path from 'node:path'; import test from 'ava'; import { EventEmitter } from 'node:events'; -import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; +import { WorkflowState } from '../../src/types'; import initWorkers from '../../src/api/call-worker'; import execute from '../../src/api/execute'; import { createMockLogger } from '@openfn/logger'; @@ -11,19 +11,19 @@ import { WORKFLOW_START, } from '../../src/events'; import { RTEOptions } from '../../src/api'; +import { ExecutionContext } from '../../src/engine'; const workerPath = path.resolve('dist/worker/mock.js'); -const createContext = ({ state, options }: Partial = {}) => { - const api = new EventEmitter(); - Object.assign(api, { - logger: createMockLogger(), +const createContext = ({ state, options }) => { + const ctx = new ExecutionContext({ state: state || {}, + logger: createMockLogger(), + callWorker: () => {}, options, - // logger: not used - }) as unknown as ExecutionContext; - initWorkers(api, workerPath); - return api; + }); + initWorkers(ctx, workerPath); + return ctx; }; const plan = { diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index ffb0b4fc3..0c1da6a75 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -1,25 +1,24 @@ import test from 'ava'; -import { EventEmitter } from 'node:events'; import * as e from '../../src/events'; import { createMockLogger } from '@openfn/logger'; import { log, workflowComplete, workflowStart } from '../../src/api/lifecycle'; -import { EngineAPI, ExecutionContext, WorkflowState } from '../../src/types'; +import { WorkflowState } from '../../src/types'; +import { ExecutionContext } from '../../src/engine'; -const createContext = ({ state }: Partial = {}) => { - const api = new EventEmitter(); - return Object.assign(api, { +const createContext = (workflowId: string, state?: any) => + new ExecutionContext({ + state: state || { id: workflowId }, logger: createMockLogger(), - state: state || {}, - // logger: not used - }) as unknown as ExecutionContext; -}; + callWorker: () => {}, + options: {}, + }); test(`workflowStart: emits ${e.WORKFLOW_START}`, (t) => { return new Promise((done) => { const workflowId = 'a'; - const context = createContext(); + const context = createContext(workflowId); const event = { workflowId, threadId: '123' }; context.on(e.WORKFLOW_START, (evt) => { @@ -34,7 +33,7 @@ test(`workflowStart: emits ${e.WORKFLOW_START}`, (t) => { test('onWorkflowStart: updates state', (t) => { const workflowId = 'a'; - const context = createContext(); + const context = createContext(workflowId); const event = { workflowId, threadId: '123' }; workflowStart(context, event); @@ -55,9 +54,10 @@ test(`workflowComplete: emits ${e.WORKFLOW_COMPLETE}`, (t) => { const result = { a: 777 }; const state = { + id: workflowId, startTime: Date.now() - 1000, } as WorkflowState; - const context = createContext({ state }); + const context = createContext(workflowId, state); const event = { workflowId, state: result, threadId: '1' }; @@ -77,9 +77,10 @@ test('workflowComplete: updates state', (t) => { const result = { a: 777 }; const state = { + id: workflowId, startTime: Date.now() - 1000, } as WorkflowState; - const context = createContext({ state }); + const context = createContext(workflowId, state); const event = { workflowId, state: result, threadId: '1' }; workflowComplete(context, event); @@ -93,10 +94,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { return new Promise((done) => { const workflowId = 'a'; - const state = { - id: workflowId, - } as WorkflowState; - const context = createContext({ state }); + const context = createContext(workflowId); const event = { workflowId, @@ -111,7 +109,7 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { context.on(e.WORKFLOW_LOG, (evt) => { t.deepEqual(evt, { - workflowId: state.id, + workflowId, threadId: 'a', ...event.message, }); From 2cb974a293732eee67868a2285e3efa7789da806 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 18:33:57 +0100 Subject: [PATCH 161/232] engine: refactor ExecutionContext into its own file --- .../src/classes/ExecutionContext.ts | 44 +++++++++++++++++++ packages/engine-multi/src/engine.ts | 38 +--------------- .../engine-multi/test/api/autoinstall.test.ts | 2 +- .../engine-multi/test/api/execute.test.ts | 2 +- .../engine-multi/test/api/lifecycle.test.ts | 2 +- 5 files changed, 49 insertions(+), 39 deletions(-) create mode 100644 packages/engine-multi/src/classes/ExecutionContext.ts diff --git a/packages/engine-multi/src/classes/ExecutionContext.ts b/packages/engine-multi/src/classes/ExecutionContext.ts new file mode 100644 index 000000000..ec5abe3ae --- /dev/null +++ b/packages/engine-multi/src/classes/ExecutionContext.ts @@ -0,0 +1,44 @@ +import { EventEmitter } from 'node:events'; + +import type { + WorkflowState, + CallWorker, + ExecutionContextConstructor, +} from '../types'; +import type { Logger } from '@openfn/logger'; +import type { EngineOptions } from '../engine'; + +/** + * The ExeuctionContext class wraps an event emitter with some useful context + * and automatically appends the workflow id to each emitted events + * + * Each running workflow has its own context object + */ + +// TODO could use some explicit unit tests on this +export default class ExecutionContext extends EventEmitter { + state: WorkflowState; + logger: Logger; + callWorker: CallWorker; + options: EngineOptions; + + constructor({ + state, + logger, + callWorker, + options, + }: ExecutionContextConstructor) { + super(); + this.logger = logger; + this.callWorker = callWorker; + this.state = state; + this.options = options; + } + + // override emit to add the workflowId to all events + // @ts-ignore + emit(event: string, payload: any) { + payload.workflowId = this.state.id; + super.emit(event, payload); + } +} diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index d97785998..64af29409 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -7,15 +7,10 @@ import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from './events'; import initWorkers from './api/call-worker'; import createState from './api/create-state'; import execute from './api/execute'; +import ExecutionContext from './classes/ExecutionContext'; import type { LazyResolvers } from './api'; -import type { - EngineAPI, - EventHandler, - WorkflowState, - CallWorker, - ExecutionContextConstructor, -} from './types'; +import type { EngineAPI, EventHandler, WorkflowState } from './types'; import { Logger } from '@openfn/logger'; import { AutoinstallOptions } from './api/autoinstall'; @@ -54,35 +49,6 @@ const createWorkflowEvents = ( // But I should probably lean in to the class more for typing and stuff class Engine extends EventEmitter {} -// TODO this is actually the api that each execution gets -// its nice to separate that from the engine a bit -export class ExecutionContext extends EventEmitter { - state: WorkflowState; - logger: Logger; - callWorker: CallWorker; - options: EngineOptions; - - constructor({ - state, - logger, - callWorker, - options, - }: ExecutionContextConstructor) { - super(); - this.logger = logger; - this.callWorker = callWorker; - this.state = state; - this.options = options; - } - - // override emit to add the workflowId to all events - // @ts-ignore - emit(event, payload) { - payload.workflowId = this.state.id; - super.emit(event, payload); - } -} - // The engine is way more strict about options export type EngineOptions = { repoDir: string; diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index 6c395a793..67933b9f4 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -3,7 +3,7 @@ import { createMockLogger } from '@openfn/logger'; import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; import { AUTOINSTALL_COMPLETE } from '../../src/events'; -import { ExecutionContext } from '../../src/engine'; +import ExecutionContext from '../../src/classes/ExecutionContext'; type PackageJson = { name: string; diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index d30e74247..42a4de926 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -11,7 +11,7 @@ import { WORKFLOW_START, } from '../../src/events'; import { RTEOptions } from '../../src/api'; -import { ExecutionContext } from '../../src/engine'; +import ExecutionContext from '../../src/classes/ExecutionContext'; const workerPath = path.resolve('dist/worker/mock.js'); diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 0c1da6a75..be1978282 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -4,7 +4,7 @@ import * as e from '../../src/events'; import { createMockLogger } from '@openfn/logger'; import { log, workflowComplete, workflowStart } from '../../src/api/lifecycle'; import { WorkflowState } from '../../src/types'; -import { ExecutionContext } from '../../src/engine'; +import ExecutionContext from '../../src/classes/ExecutionContext'; const createContext = (workflowId: string, state?: any) => new ExecutionContext({ From 2efbb06d7d6f7164ef145dc4302a0e8d4c81add2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 19:05:42 +0100 Subject: [PATCH 162/232] worker-tests: add autoinstall test --- integration-tests/worker/package.json | 5 +- .../worker/test/integration.test.ts | 65 ++++++++++++++++++- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index bb6d9d1e4..1dad0ad41 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -7,12 +7,11 @@ "license": "ISC", "type": "module", "scripts": { - "clean": "rimraf dist repo", + "clean": "rimraf dist tmp/repo/*", "build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/worker/dist --no-version", "build": "pnpm build:pack && docker build --tag worker-integration-tests .", "start": "docker run worker-integration-tests", - "test": "npx ava -s --timeout 2m && pnpm clean", - "test:dev": "pnpm ava -s && pnpm clean" + "test": "pnpm clean && npx ava -s --timeout 2m" }, "dependencies": { "@openfn/engine-multi": "workspace:^", diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 6e9935265..e57775f2d 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -1,4 +1,5 @@ import test from 'ava'; +import path from 'node:path'; import createLightningServer from '@openfn/lightning-mock'; @@ -9,6 +10,7 @@ import createLogger, { createMockLogger } from '@openfn/logger'; let lightning; let worker; +let engine; test.afterEach(() => { lightning.destroy(); @@ -20,10 +22,11 @@ const initLightning = () => { }; const initWorker = () => { - const engine = createEngine({ + engine = createEngine({ // logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), runtimeLogger: createMockLogger(), + repoDir: path.resolve('./tmp/repo'), }); worker = createWorkerServer(engine, { @@ -63,6 +66,11 @@ test.only('should run a simple job with no compilation', (t) => { return new Promise((done) => { initLightning(); lightning.on('attempt:complete', (evt) => { + // the complete payload should have a final dataclip id + // we should also be able to ask lightning for the result + // const { payload: state } = evt; + // console.log(evt); + // t.deepEqual(state, { data: { answer: 42 } }); t.pass('completed attempt'); done(); }); @@ -79,3 +87,58 @@ test.only('should run a simple job with no compilation', (t) => { }); }); }); + +// todo ensure repo is clean +// check how we manage the env in cli tests +test('run a job with autoinstall of common', (t) => { + return new Promise((done) => { + initLightning(); + + let autoinstallEvent; + + lightning.on('attempt:complete', (evt) => { + try { + t.truthy(autoinstallEvent); + t.is(autoinstallEvent.module, '@openfn/language-common'); + t.is(autoinstallEvent.version, 'latest'); + t.assert(autoinstallEvent.duration >= 100); + + const { result } = evt; + t.deepEqual(result, { data: { answer: 42 } }); + done(); + } catch (e) { + t.fail(e); + done(); + } + }); + + initWorker(); + + // listen to events for this attempt + engine.listen('a1', { + 'autoinstall-complete': (evt) => { + autoinstallEvent = evt; + }, + }); + + lightning.enqueueAttempt({ + id: 'a1', + jobs: [ + { + id: 'j1', + adaptor: '@openfn/language-common@latest', // version lock to something stable? + body: 'fn(() => ({ data: { answer: 42} }))', + }, + ], + }); + }); +}); + +// this depends on prior test! +test.todo("run a job which doesn't autoinstall common"); +test.todo('run a job with complex behaviours (initial state, branching)'); + +// maybe create a http server with basic auth on the endpoint +// use http adaptor to call the server +// obviously credential is lazy loaded +test.todo('run a job which requires credentials'); From d736b46b62fe49a151b72bf12043d80f8e5975e2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 19:18:53 +0100 Subject: [PATCH 163/232] runtime: update job start and complete events --- packages/runtime/src/execute/expression.ts | 11 ++++++++--- packages/runtime/src/execute/job.ts | 2 +- packages/runtime/test/execute/expression.test.ts | 16 +++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts index 788dbcf2e..10478aea6 100644 --- a/packages/runtime/src/execute/expression.ts +++ b/packages/runtime/src/execute/expression.ts @@ -10,7 +10,8 @@ import clone from '../util/clone'; export default ( ctx: ExecutionContext, expression: string | Operation[], - initialState: State + initialState: State, + id?: string ) => new Promise(async (resolve, reject) => { const { logger, notify = () => {}, opts = {} } = ctx; @@ -34,7 +35,7 @@ export default ( try { logger.debug(`Executing expression (${operations.length} operations)`); let exeDuration = Date.now(); - notify('job-start'); + notify('job-start', { jobId: id }); const tid = setTimeout(() => { logger.error(`Error: Timeout (${timeout}ms) expired!`); @@ -51,7 +52,11 @@ export default ( exeDuration = Date.now() - exeDuration; - notify('job-complete', { duration: exeDuration, state: result }); + notify('job-complete', { + duration: exeDuration, + state: result, + jobId: id, + }); // return the final state resolve(prepareFinalState(opts, result)); diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts index 0471ff109..dafddbae1 100644 --- a/packages/runtime/src/execute/job.ts +++ b/packages/runtime/src/execute/job.ts @@ -80,7 +80,7 @@ const executeJob = async ( if (job.expression) { // The expression SHOULD return state, but could return anything try { - result = await executeExpression(ctx, job.expression, state); + result = await executeExpression(ctx, job.expression, state, job.id); const duration = logger.timer('job'); logger.success(`Completed job ${job.id} in ${duration}`); } catch (e: any) { diff --git a/packages/runtime/test/execute/expression.test.ts b/packages/runtime/test/execute/expression.test.ts index 8cdab5588..941aae4ad 100644 --- a/packages/runtime/test/execute/expression.test.ts +++ b/packages/runtime/test/execute/expression.test.ts @@ -69,39 +69,41 @@ test('run a live no-op job with @openfn/language-common.fn', async (t) => { test('notify job-start', async (t) => { let didCallCallback = false; - const job = [(s: State) => s]; + const expression = [(s: State) => s]; const state = createState(); - const notify = (event: string, _payload?: any) => { + const notify = (event: string, payload?: any) => { if (event === 'job-start') { didCallCallback = true; } + t.is(payload.jobId, 'j'); }; const context = createContext({ notify }); - await execute(context, job, state); + await execute(context, expression, state, 'j'); t.true(didCallCallback); }); -test('call the on-complete callback', async (t) => { +test('notifu job-complete', async (t) => { let didCallCallback = false; - const job = [(s: State) => s]; + const expression = [(s: State) => s]; const state = createState(); const notify = (event: string, payload: any) => { if (event === 'job-complete') { - const { state, duration } = payload; + const { state, duration, jobId } = payload; didCallCallback = true; t.truthy(state); t.assert(!isNaN(duration)); + t.is(jobId, 'j'); } }; const context = createContext({ notify }); - await execute(context, job, state); + await execute(context, expression, state, 'j'); t.true(didCallCallback); }); From f5cf94f9833f8d1ce828d2e6dc510eb241cef4e4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 17 Oct 2023 19:41:56 +0100 Subject: [PATCH 164/232] engine: map job start and complete events properly --- packages/engine-multi/src/api/execute.ts | 14 +++- packages/engine-multi/src/api/lifecycle.ts | 29 +++++++- packages/engine-multi/src/worker/events.ts | 10 ++- .../engine-multi/src/worker/mock-worker.ts | 10 ++- .../engine-multi/src/worker/worker-helper.ts | 35 ++++++---- packages/engine-multi/src/worker/worker.ts | 13 +++- .../engine-multi/test/api/execute.test.ts | 46 ++++++++++++- .../engine-multi/test/api/lifecycle.test.ts | 68 ++++++++++++++++++- .../engine-multi/test/integration.test.ts | 5 +- 9 files changed, 202 insertions(+), 28 deletions(-) diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 3cbee1571..e5cdfa851 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -4,7 +4,13 @@ import { ExecutionContext } from '../types'; import autoinstall from './autoinstall'; import compile from './compile'; -import { workflowStart, workflowComplete, log } from './lifecycle'; +import { + workflowStart, + workflowComplete, + log, + jobStart, + jobComplete, +} from './lifecycle'; const execute = async (context: ExecutionContext) => { const { state, callWorker, logger } = context; @@ -20,6 +26,12 @@ const execute = async (context: ExecutionContext) => { ) => { workflowComplete(context, evt); }, + [workerEvents.JOB_START]: (evt: workerEvents.JobStartEvent) => { + jobStart(context, evt); + }, + [workerEvents.JOB_COMPLETE]: (evt: workerEvents.JobCompleteEvent) => { + jobComplete(context, evt); + }, [workerEvents.LOG]: (evt: workerEvents.LogEvent) => { log(context, evt); }, diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index a2adc972d..327ab6260 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -68,6 +68,32 @@ export const workflowComplete = ( }); }; +export const jobStart = ( + context: ExecutionContext, + event: internalEvents.JobStartEvent +) => { + const { threadId, jobId } = event; + + context.emit(externalEvents.JOB_START, { + jobId, + threadId, + }); +}; + +export const jobComplete = ( + context: ExecutionContext, + event: internalEvents.JobCompleteEvent +) => { + const { threadId, state, duration, jobId } = event; + + context.emit(externalEvents.JOB_COMPLETE, { + threadId, + state, + duration, + jobId, + }); +}; + export const log = ( context: ExecutionContext, event: internalEvents.LogEvent @@ -89,6 +115,3 @@ export const log = ( ...event.message, }); }; - -// TODO jobstart -// TODO jobcomplete diff --git a/packages/engine-multi/src/worker/events.ts b/packages/engine-multi/src/worker/events.ts index 0e3f4b3bc..8e0313100 100644 --- a/packages/engine-multi/src/worker/events.ts +++ b/packages/engine-multi/src/worker/events.ts @@ -34,9 +34,15 @@ export interface WorkflowCompleteEvent extends InternalEvent { state: any; } -export interface JobStartEvent extends InternalEvent {} +export interface JobStartEvent extends InternalEvent { + jobId: string; +} -export interface JobCompleteEvent extends InternalEvent {} +export interface JobCompleteEvent extends InternalEvent { + jobId: string; + state: any; + duration: number; +} export interface LogEvent extends InternalEvent { message: JSONLog; diff --git a/packages/engine-multi/src/worker/mock-worker.ts b/packages/engine-multi/src/worker/mock-worker.ts index e973573f5..551980386 100644 --- a/packages/engine-multi/src/worker/mock-worker.ts +++ b/packages/engine-multi/src/worker/mock-worker.ts @@ -8,7 +8,8 @@ * and reading instructions out of state object. */ import workerpool from 'workerpool'; -import helper, { createLoggers } from './worker-helper'; +import helper, { createLoggers, publish } from './worker-helper'; +import * as workerEvents from './events'; type MockJob = { id?: string; @@ -33,7 +34,9 @@ function mock(plan: MockExecutionPlan) { const [job] = plan.jobs; const { jobLogger } = createLoggers(plan.id!); return new Promise((resolve) => { + const jobId = job.id || ''; setTimeout(async () => { + publish(plan.id, workerEvents.JOB_START, { jobId }); // TODO this isn't data, but state - it's the whole state object (minus config) let state: any = { data: job.data || {} }; if (job.expression) { @@ -57,6 +60,11 @@ function mock(plan: MockExecutionPlan) { }; } } + publish(plan.id, workerEvents.JOB_COMPLETE, { + jobId, + duration: 100, + state, + }); resolve(state); }, job._delay || 1); }); diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts index 852619cc8..d09ab7349 100644 --- a/packages/engine-multi/src/worker/worker-helper.ts +++ b/packages/engine-multi/src/worker/worker-helper.ts @@ -42,35 +42,40 @@ export const createLoggers = (workflowId: string) => { return { logger, jobLogger }; }; +export function publish( + workflowId: string, + type: T, + payload: Omit +) { + workerpool.workerEmit({ + workflowId, + threadId, + type, + ...payload, + }); +} + // TODO use bespoke event names here // maybe thread:workflow-start async function helper(workflowId: string, execute: () => Promise) { - function publish( - type: T, - payload: Omit - ) { - workerpool.workerEmit({ - workflowId, - threadId, - type, - ...payload, - }); - } - - publish(workerEvents.WORKFLOW_START, {}); + publish(workflowId, workerEvents.WORKFLOW_START, {}); try { // Note that the worker thread may fire logs after completion // I think this is fine, it's just a log stream thing // But the output is very confusing! const result = await execute(); - publish(workerEvents.WORKFLOW_COMPLETE, { state: result }); + publish(workflowId, workerEvents.WORKFLOW_COMPLETE, { state: result }); // For tests return result; } catch (err: any) { console.error(err); - publish(workerEvents.ERROR, { workflowId, threadId, message: err.message }); + publish(workflowId, workerEvents.ERROR, { + workflowId, + threadId, + message: err.message, + }); } } diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 6b67d107b..8dd07c165 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -13,7 +13,8 @@ import workerpool from 'workerpool'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; -import helper, { createLoggers } from './worker-helper'; +import helper, { createLoggers, publish } from './worker-helper'; +import type { WorkerEvents } from './events'; workerpool.worker({ // TODO: add a startup script to ensure the worker is ok @@ -31,6 +32,16 @@ workerpool.worker({ linker: { modules: adaptorPaths, }, + callbacks: { + // TODO load state (maybe) + // TODO load credential + notify: (name: WorkerEvents, payload: any) => { + // Event handling here is a bit sketchy + // the runtime will publish eg job-start, but we need to map it to worker:job-start + // @ts-ignore + publish(plan.id, `worker:${name}`, payload); + }, + }, }; return helper(plan.id!, () => run(plan, {}, options)); diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index 42a4de926..a08d0b411 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -6,6 +6,8 @@ import initWorkers from '../../src/api/call-worker'; import execute from '../../src/api/execute'; import { createMockLogger } from '@openfn/logger'; import { + JOB_COMPLETE, + JOB_START, WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START, @@ -17,7 +19,7 @@ const workerPath = path.resolve('dist/worker/mock.js'); const createContext = ({ state, options }) => { const ctx = new ExecutionContext({ - state: state || {}, + state: state || { workflowId: 'x' }, logger: createMockLogger(), callWorker: () => {}, options, @@ -30,6 +32,7 @@ const plan = { id: 'x', jobs: [ { + id: 'j', // this will basically be evalled expression: '() => 22', }, @@ -46,6 +49,7 @@ const options = { test.serial('execute should run a job and return the result', async (t) => { const state = { + id: 'x', plan, } as WorkflowState; @@ -58,6 +62,7 @@ test.serial('execute should run a job and return the result', async (t) => { // we can check the state object after each of these is returned test.serial('should emit a workflow-start event', async (t) => { const state = { + id: 'x', plan, } as WorkflowState; let workflowStart; @@ -75,6 +80,7 @@ test.serial('should emit a workflow-start event', async (t) => { test.serial('should emit a workflow-complete event', async (t) => { let workflowComplete; const state = { + id: 'x', plan, } as WorkflowState; @@ -88,7 +94,43 @@ test.serial('should emit a workflow-complete event', async (t) => { t.is(workflowComplete.state, 22); }); -test.serial.only('should emit a log event', async (t) => { +test.serial('should emit a job-start event', async (t) => { + const state = { + id: 'x', + plan, + } as WorkflowState; + + let event; + + const context = createContext({ state, options }); + + context.once(JOB_START, (evt) => (event = evt)); + + await execute(context); + + t.is(event.jobId, 'j'); +}); + +test.serial('should emit a job-complete event', async (t) => { + const state = { + id: 'x', + plan, + } as WorkflowState; + + let event; + + const context = createContext({ state, options }); + + context.once(JOB_COMPLETE, (evt) => (event = evt)); + + await execute(context); + + t.is(event.jobId, 'j'); + t.is(event.state, 22); + t.assert(!isNaN(event.duration)); +}); + +test.serial('should emit a log event', async (t) => { let workflowLog; const plan = { id: 'y', diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index be1978282..9b692d0f5 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -2,7 +2,13 @@ import test from 'ava'; import * as e from '../../src/events'; import { createMockLogger } from '@openfn/logger'; -import { log, workflowComplete, workflowStart } from '../../src/api/lifecycle'; +import { + log, + workflowComplete, + workflowStart, + jobStart, + jobComplete, +} from '../../src/api/lifecycle'; import { WorkflowState } from '../../src/types'; import ExecutionContext from '../../src/classes/ExecutionContext'; @@ -90,6 +96,66 @@ test('workflowComplete: updates state', (t) => { t.deepEqual(state.result, result); }); +test(`job-start: emits ${e.JOB_START}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + + const state = { + id: workflowId, + startTime: Date.now() - 1000, + } as WorkflowState; + + const context = createContext(workflowId, state); + + const event = { + workflowId, + threadId: '1', + jobId: 'j', + }; + + context.on(e.JOB_START, (evt) => { + t.is(evt.workflowId, workflowId); + t.is(evt.threadId, '1'); + t.is(evt.jobId, 'j'); + done(); + }); + + jobStart(context, event); + }); +}); + +test(`job-complete: emits ${e.JOB_COMPLETE}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + + const state = { + id: workflowId, + startTime: Date.now() - 1000, + } as WorkflowState; + + const context = createContext(workflowId, state); + + const event = { + workflowId, + threadId: '1', + jobId: 'j', + duration: 200, + state: 22, + }; + + context.on(e.JOB_COMPLETE, (evt) => { + t.is(evt.workflowId, workflowId); + t.is(evt.threadId, '1'); + t.is(evt.jobId, 'j'); + t.is(evt.state, 22); + t.is(evt.duration, 200); + done(); + }); + + jobComplete(context, event); + }); +}); + test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { return new Promise((done) => { const workflowId = 'a'; diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 4145b8803..c24d27845 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -24,6 +24,7 @@ const createPlan = (jobs?: any[]) => ({ id: `${++idgen}`, jobs: jobs || [ { + id: 'j1', expression: 'export default [s => s]', }, ], @@ -49,7 +50,7 @@ test('trigger workflow-start', (t) => { }); }); -test.skip('trigger job-start', (t) => { +test('trigger job-start', (t) => { return new Promise((done) => { const api = createAPI({ logger, @@ -67,7 +68,7 @@ test.skip('trigger job-start', (t) => { }); }); -test.skip('trigger job-complete', (t) => { +test('trigger job-complete', (t) => { return new Promise((done) => { const api = createAPI({ logger, From 08c44bb9638234e16ee0270ddde949c815169902 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 11:50:23 +0100 Subject: [PATCH 165/232] lightning-mock: update tests Fix an issue where events wasnt passed into socket tests, and add a getResult test --- .../lightning-mock/test/lightning.test.ts | 24 +++++++++++++++++++ .../lightning-mock/test/socket-server.test.ts | 5 +++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/lightning-mock/test/lightning.test.ts b/packages/lightning-mock/test/lightning.test.ts index 60b5cc798..6a40b21eb 100644 --- a/packages/lightning-mock/test/lightning.test.ts +++ b/packages/lightning-mock/test/lightning.test.ts @@ -342,4 +342,28 @@ test.serial( } ); +// TODO this should probably return reason AND state? +test.serial( + 'getResult should return the correct resulting dataclip', + async (t) => { + return new Promise(async (done) => { + const result = { answer: 42 }; + + server.startAttempt(attempt1.id); + server.addDataclip('result', result); + + server.waitForResult(attempt1.id).then(() => { + const dataclip = server.getResult(attempt1.id); + t.deepEqual(result, dataclip); + done(); + }); + + const channel = await join(`attempt:${attempt1.id}`, { token: 'a.b.c' }); + channel.push(ATTEMPT_COMPLETE, { + final_dataclip_id: 'result', + } as AttemptCompletePayload); + }); + } +); + // test.serial('getLogs should return logs', async (t) => {}); diff --git a/packages/lightning-mock/test/socket-server.test.ts b/packages/lightning-mock/test/socket-server.test.ts index 7636f13e2..d0fc34e0c 100644 --- a/packages/lightning-mock/test/socket-server.test.ts +++ b/packages/lightning-mock/test/socket-server.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; +import EventEmitter from 'node:events'; import { Socket } from 'phoenix'; import { WebSocket } from 'ws'; - import createSocketServer from '../src/socket-server'; let socket; @@ -19,6 +19,9 @@ test.beforeEach( messages = []; // @ts-ignore I don't care about missing server options here server = createSocketServer({ + state: { + events: new EventEmitter(), + }, onMessage: (evt) => { messages.push(evt); }, From 8244663cfebe37f861b8f838d93c7614dd73291c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 12:43:45 +0100 Subject: [PATCH 166/232] runtime: be less eager to default initial state Basically if the incoming plan has initial state, the runtime should not override it --- packages/runtime/src/execute/plan.ts | 1 - packages/runtime/src/runtime.ts | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/runtime/src/execute/plan.ts b/packages/runtime/src/execute/plan.ts index 1b28f0c9c..5aebdf2e5 100644 --- a/packages/runtime/src/execute/plan.ts +++ b/packages/runtime/src/execute/plan.ts @@ -39,7 +39,6 @@ const executePlan = async ( const leaves: Record = {}; let { initialState } = compiledPlan; - if (typeof initialState === 'string') { const id = initialState; const startTime = Date.now(); diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 3ecaf7be2..77ae646fc 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -47,7 +47,7 @@ const defaultLogger = createMockLogger(); // so maybe state becomes an option in the opts object const run = ( expressionOrXPlan: string | Operation[] | ExecutionPlan, - state: State = defaultState, + state: State, opts: Options = {} ) => { const logger = opts.logger || defaultLogger; @@ -81,8 +81,11 @@ const run = ( } else { plan = expressionOrXPlan as ExecutionPlan; } + if (state) { plan.initialState = clone(state); + } else if (!plan.initialState) { + plan.initialState = defaultState; } return executePlan(plan, opts, logger); From bc01d18c307200da9a4a4e3d7dc6f50c42859a2a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 12:44:24 +0100 Subject: [PATCH 167/232] various: update worker integration tests Initial state test --- .../worker/test/integration.test.ts | 143 ++++++++++++++++-- packages/engine-multi/src/api/execute.ts | 1 - packages/engine-multi/src/engine.ts | 16 +- packages/engine-multi/src/worker/worker.ts | 1 + packages/ws-worker/src/api/execute.ts | 12 +- packages/ws-worker/src/server.ts | 5 +- 6 files changed, 155 insertions(+), 23 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index e57775f2d..09ef66bd9 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -1,5 +1,6 @@ import test from 'ava'; import path from 'node:path'; +import crypto from 'node:crypto'; import createLightningServer from '@openfn/lightning-mock'; @@ -18,6 +19,8 @@ test.afterEach(() => { }); const initLightning = () => { + // TODO the lightning mock right now doesn't use the secret + // but we may want to add tests against this lightning = createLightningServer({ port: 9999 }); }; @@ -34,7 +37,7 @@ const initWorker = () => { // logger: createLogger('worker', { level: 'debug' }), port: 2222, lightning: 'ws://localhost:9999/worker', - secret: 'abc', // TODO use a more realistic secret + secret: crypto.randomUUID(), }); }; @@ -62,15 +65,14 @@ test('should join attempts queue channel', (t) => { }); }); -test.only('should run a simple job with no compilation', (t) => { +test('should run a simple job with no compilation', (t) => { return new Promise((done) => { initLightning(); lightning.on('attempt:complete', (evt) => { - // the complete payload should have a final dataclip id - // we should also be able to ask lightning for the result - // const { payload: state } = evt; - // console.log(evt); - // t.deepEqual(state, { data: { answer: 42 } }); + // This will fetch the final dataclip from the attempt + const result = lightning.getResult('a1'); + t.deepEqual(result, { data: { answer: 42 } }); + t.pass('completed attempt'); done(); }); @@ -101,10 +103,13 @@ test('run a job with autoinstall of common', (t) => { t.truthy(autoinstallEvent); t.is(autoinstallEvent.module, '@openfn/language-common'); t.is(autoinstallEvent.version, 'latest'); - t.assert(autoinstallEvent.duration >= 100); + // Expect autoinstall to take several seconds + t.assert(autoinstallEvent.duration >= 1000); - const { result } = evt; + // This will fetch the final dataclip from the attempt + const result = lightning.getResult('a33'); t.deepEqual(result, { data: { answer: 42 } }); + done(); } catch (e) { t.fail(e); @@ -115,14 +120,14 @@ test('run a job with autoinstall of common', (t) => { initWorker(); // listen to events for this attempt - engine.listen('a1', { + engine.listen('a33', { 'autoinstall-complete': (evt) => { autoinstallEvent = evt; }, }); lightning.enqueueAttempt({ - id: 'a1', + id: 'a33', jobs: [ { id: 'j1', @@ -135,8 +140,120 @@ test('run a job with autoinstall of common', (t) => { }); // this depends on prior test! -test.todo("run a job which doesn't autoinstall common"); -test.todo('run a job with complex behaviours (initial state, branching)'); +test('run a job which does NOT autoinstall common', (t) => { + return new Promise((done, _fail) => { + initLightning(); + + lightning.on('attempt:complete', (evt) => { + try { + // This will fetch the final dataclip from the attempt + const result = lightning.getResult('a10'); + t.deepEqual(result, { data: { answer: 42 } }); + + done(); + } catch (e) { + t.fail(e); + done(); + } + }); + + initWorker(); + + // listen to events for this attempt + engine.listen('a10', { + 'autoinstall-complete': (evt) => { + // TODO: I think soon I'm going to issue a compelte event even if + // it loads from cache, so this will need changing + t.fail('Unexpeted autoinstall event!'); + }, + }); + + lightning.enqueueAttempt({ + id: 'a10', + jobs: [ + { + id: 'j1', + adaptor: '@openfn/language-common@latest', // version lock to something stable? + body: 'fn(() => ({ data: { answer: 42} }))', + }, + ], + }); + }); +}); + +test('run a job with initial state', (t) => { + return new Promise((done) => { + const attempt = { + id: crypto.randomUUID(), + dataclip_id: 's1', + jobs: [ + { + adaptor: '@openfn/language-common@latest', + body: 'fn((s) => s)', + }, + ], + }; + + initLightning(); + + const initialState = { data: { name: 'Professor X' } }; + + lightning.addDataclip('s1', initialState); + + lightning.on('attempt:complete', () => { + const result = lightning.getResult(attempt.id); + t.deepEqual(result, { + ...initialState, + configuration: {}, + }); + done(); + }); + + initWorker(); + + // TODO: is there any way I can test the worker behaviour here? + // I think I can listen to load-state right? + // well, not really, not yet, not from the worker + // see https://github.com/OpenFn/kit/issues/402 + + lightning.enqueueAttempt(attempt); + }); +}); + +// test('run a job with complex behaviours (initial state, branching)', (t) => { +// const attempt = { +// id: 'a1', +// initialState: 's1 +// jobs: [ +// { +// id: 'j1', +// body: 'const fn = (f) => (state) => f(state); fn(() => ({ data: { answer: 42} }))', +// }, +// ], +// } + +// initLightning(); +// lightning.on('attempt:complete', (evt) => { +// // This will fetch the final dataclip from the attempt +// const result = lightning.getResult('a1'); +// t.deepEqual(result, { data: { answer: 42 } }); + +// t.pass('completed attempt'); +// done(); +// }); +// initWorker(); + +// lightning.enqueueAttempt({ +// id: 'a1', +// jobs: [ +// { +// id: 'j1', +// body: 'const fn = (f) => (state) => f(state); fn(() => ({ data: { answer: 42} }))', +// }, +// ], +// }); +// }); +// }); // maybe create a http server with basic auth on the endpoint // use http adaptor to call the server diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index e5cdfa851..4aaefc64e 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -36,7 +36,6 @@ const execute = async (context: ExecutionContext) => { log(context, evt); }, }; - return callWorker('run', [state.plan, adaptorPaths], events).catch( (e: any) => { // TODO what about errors then? diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 64af29409..9f0373374 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -3,7 +3,13 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import type { ExecutionPlan } from '@openfn/runtime'; -import { WORKFLOW_COMPLETE, WORKFLOW_LOG, WORKFLOW_START } from './events'; +import { + JOB_COMPLETE, + JOB_START, + WORKFLOW_COMPLETE, + WORKFLOW_LOG, + WORKFLOW_START, +} from './events'; import initWorkers from './api/call-worker'; import createState from './api/create-state'; import execute from './api/execute'; @@ -11,8 +17,8 @@ import ExecutionContext from './classes/ExecutionContext'; import type { LazyResolvers } from './api'; import type { EngineAPI, EventHandler, WorkflowState } from './types'; -import { Logger } from '@openfn/logger'; -import { AutoinstallOptions } from './api/autoinstall'; +import type { Logger } from '@openfn/logger'; +import type { AutoinstallOptions } from './api/autoinstall'; // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? @@ -37,8 +43,8 @@ const createWorkflowEvents = ( } proxy(WORKFLOW_START); proxy(WORKFLOW_COMPLETE); - // proxy(JOB_START); - // proxy(JOB_COMPLETE); + proxy(JOB_START); + proxy(JOB_COMPLETE); proxy(WORKFLOW_LOG); return context; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 8dd07c165..ceffcd4f2 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -44,6 +44,7 @@ workerpool.worker({ }, }; + // @ts-ignore options events return helper(plan.id!, () => run(plan, {}, options)); }, }); diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 1e268e09b..3429d29b3 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -117,22 +117,30 @@ export function execute( addEvent('log', onJobLog), // This will also resolve the promise addEvent('workflow-complete', onWorkflowComplete) + + // TODO send autoinstall logs + // are these associated with a workflow...? + // well, I guess they can be! + // Or is this just a log? + // Or a generic metric? ); engine.listen(plan.id, listeners); const resolvers = { credential: (id: string) => loadCredential(channel, id), - // dataclip: (id: string) => loadDataclip(channel, id), + // TODO not supported right now + // dataclip: (id: string) => loadDataclip(channel, id), }; + // TODO we nede to remove this from here nad let the runtime take care of it through + // the resolver. See https://github.com/OpenFn/kit/issues/403 if (typeof plan.initialState === 'string') { logger.debug('loading dataclip', plan.initialState); plan.initialState = await loadDataclip(channel, plan.initialState); logger.success('dataclip loaded'); logger.debug(plan.initialState); } - engine.execute(plan, resolvers); }); } diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 74baa1743..b0d690713 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -120,8 +120,9 @@ function createServer(engine: any, options: ServerOptions = {}) { } // TMP doing this for tests but maybe its better done externally - app.on = (...args) => engine.on(...args); - app.once = (...args) => engine.once(...args); + app.on = (...args) => { + return engine.on(...args); + }; return app; } From 7136bc8b9e9546316eec4cb21df767b9e427b5f0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 14:39:44 +0100 Subject: [PATCH 168/232] integration-tests: dockerise worker tests, update command to run multiples suites --- integration-tests/worker/.dockerignore | 2 ++ integration-tests/worker/Dockerfile | 15 ++++++++ integration-tests/worker/ava.config.js | 13 +++++++ integration-tests/worker/package.json | 6 ++-- .../worker/package.json.container | 34 +++++++++++++++++++ package.json | 2 +- packages/lightning-mock/package.json | 2 +- 7 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 integration-tests/worker/.dockerignore create mode 100644 integration-tests/worker/Dockerfile create mode 100644 integration-tests/worker/ava.config.js create mode 100644 integration-tests/worker/package.json.container diff --git a/integration-tests/worker/.dockerignore b/integration-tests/worker/.dockerignore new file mode 100644 index 000000000..c8c6ad9ba --- /dev/null +++ b/integration-tests/worker/.dockerignore @@ -0,0 +1,2 @@ +node_modules +repo \ No newline at end of file diff --git a/integration-tests/worker/Dockerfile b/integration-tests/worker/Dockerfile new file mode 100644 index 000000000..c1b0072bc --- /dev/null +++ b/integration-tests/worker/Dockerfile @@ -0,0 +1,15 @@ +# This dockerfile allows us to run tests against the build in isolation +# (I'm not really sure we need it but it's been a useful dev tool) +FROM node:18.12.0 + +ENV NODE_ENV=production + +WORKDIR /app + +COPY . . + +COPY ./package.json.container ./package.json + +RUN npm install --production + +CMD ["npm", "test"] \ No newline at end of file diff --git a/integration-tests/worker/ava.config.js b/integration-tests/worker/ava.config.js new file mode 100644 index 000000000..0852e9508 --- /dev/null +++ b/integration-tests/worker/ava.config.js @@ -0,0 +1,13 @@ +export default { + extensions: { + ts: 'module', + }, + + environmentVariables: { + TS_NODE_TRANSPILE_ONLY: 'true', + }, + + nodeArguments: ['--loader=ts-node/esm', '--no-warnings'], + + files: ['test/**/*test.ts'], +}; diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 1dad0ad41..ce5e6c12b 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -19,6 +19,7 @@ "@openfn/logger": "workspace:^", "@openfn/ws-worker": "workspace:^", "@types/node": "^18.15.13", + "@types/rimraf": "^3.0.2", "ava": "5.3.1", "date-fns": "^2.30.0", "rimraf": "^3.0.2", @@ -29,8 +30,5 @@ "files": [ "dist", "README.md" - ], - "devDependencies": { - "@types/rimraf": "^3.0.2" - } + ] } diff --git a/integration-tests/worker/package.json.container b/integration-tests/worker/package.json.container new file mode 100644 index 000000000..8ee4bbba1 --- /dev/null +++ b/integration-tests/worker/package.json.container @@ -0,0 +1,34 @@ +{ + "name": "@openfn/integration-tests-worker", + "private": true, + "version": "1.0.0", + "description": "Lightning WOrker integration tests", + "author": "Open Function Group ", + "license": "ISC", + "type": "module", + "scripts": { + "clean": "rimraf dist tmp/repo/*", + "build:pack": "npm run clean && cd ../.. && npm run pack:local integration-tests/worker/dist --no-version", + "build": "npm run build:pack && docker build --tag worker-integration-tests .", + "start": "docker run worker-integration-tests", + "test": "npm run clean && npx ava -s --timeout 2m" + }, + "dependencies": { + "@openfn/engine-multi": "./dist/openfn-engine-multi.tgz", + "@openfn/lightning-mock": "./dist/openfn-lightning-mock.tgz", + "@openfn/logger": "./dist/openfn-logger.tgz", + "@openfn/ws-worker": "./dist/openfn-ws-worker.tgz", + "@types/node": "^18.15.13", + "ava": "5.3.1", + "date-fns": "^2.30.0", + "rimraf": "^3.0.2", + "ts-node": "10.8.1", + "tslib": "^2.4.0", + "typescript": "^5.1.6", + "@types/rimraf": "^3.0.2" + }, + "files": [ + "dist", + "README.md" + ] +} diff --git a/package.json b/package.json index 89e5832c8..12dc530b1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "pack:local": "pnpm run pack && node ./build/pack-local.js", "pack": "pnpm -r run pack", "setup": "rm -rf node_modules && rm -rf ./packages/*/node_modules && pnpm i", - "test:integration": "pnpm -C integration-tests/cli test", + "test:integration": "pnpm -r --filter=./integration-tests/* run test", "test:types": "pnpm -r --filter=./packages/* run test:types", "test": "pnpm -r --filter=./packages/* run test", "typesync": "pnpm exec typesync && pnpm -r exec typesync && pnpm install" diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 352289b2f..c32733291 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -11,7 +11,7 @@ "build": "tsup --config ../../tsup.config.js src/index.ts --no-splitting", "build:watch": "pnpm build --watch", "start": "ts-node-esm --transpile-only src/start.ts", - "_pack": "pnpm pack --pack-destination ../../dist" + "pack": "pnpm pack --pack-destination ../../dist" }, "author": "Open Function Group ", "license": "ISC", From fec240581cf4032b4ec7d35417c4dd4ec36200f2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 14:55:25 +0100 Subject: [PATCH 169/232] various: fix initial state in worker, upate typings --- integration-tests/worker/test/integration.test.ts | 3 +-- packages/engine-multi/src/events.ts | 1 + packages/engine-multi/src/worker/events.ts | 5 +++++ packages/engine-multi/src/worker/worker.ts | 15 +++++---------- packages/runtime/src/runtime.ts | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 09ef66bd9..0d2d2bf56 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -28,7 +28,6 @@ const initWorker = () => { engine = createEngine({ // logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), - runtimeLogger: createMockLogger(), repoDir: path.resolve('./tmp/repo'), }); @@ -181,7 +180,7 @@ test('run a job which does NOT autoinstall common', (t) => { }); }); -test('run a job with initial state', (t) => { +test.only('run a job with initial state', (t) => { return new Promise((done) => { const attempt = { id: crypto.randomUUID(), diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index 53ce9dd7d..cba5e699d 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -60,6 +60,7 @@ export interface JobStartPayload extends ExternalEvent { export interface JobCompletePayload extends ExternalEvent { jobId: string; + duration: number; state: any; // the result state } diff --git a/packages/engine-multi/src/worker/events.ts b/packages/engine-multi/src/worker/events.ts index 8e0313100..1aa829cf3 100644 --- a/packages/engine-multi/src/worker/events.ts +++ b/packages/engine-multi/src/worker/events.ts @@ -60,6 +60,11 @@ export type EventMap = { [JOB_COMPLETE]: JobCompleteEvent; [LOG]: LogEvent; [ERROR]: ErrorEvent; + + // TOO - extra events that aren't really designed yet + ['worker:init-start']: any; + ['worker:init-complete']: any; + ['worker:load-state']: any; }; export type WorkerEvents = keyof EventMap; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index ceffcd4f2..3ffb7c016 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -14,7 +14,7 @@ import workerpool from 'workerpool'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; import helper, { createLoggers, publish } from './worker-helper'; -import type { WorkerEvents } from './events'; +import { NotifyEvents } from '@openfn/runtime'; workerpool.worker({ // TODO: add a startup script to ensure the worker is ok @@ -33,18 +33,13 @@ workerpool.worker({ modules: adaptorPaths, }, callbacks: { - // TODO load state (maybe) - // TODO load credential - notify: (name: WorkerEvents, payload: any) => { - // Event handling here is a bit sketchy - // the runtime will publish eg job-start, but we need to map it to worker:job-start - // @ts-ignore - publish(plan.id, `worker:${name}`, payload); + notify: (name: NotifyEvents, payload: any) => { + // convert runtime notify events to internal engine events + publish(plan.id!, `worker:${name}`, payload); }, }, }; - // @ts-ignore options events - return helper(plan.id!, () => run(plan, {}, options)); + return helper(plan.id!, () => run(plan, undefined, options)); }, }); diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 77ae646fc..8743ce20d 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -47,7 +47,7 @@ const defaultLogger = createMockLogger(); // so maybe state becomes an option in the opts object const run = ( expressionOrXPlan: string | Operation[] | ExecutionPlan, - state: State, + state?: State, opts: Options = {} ) => { const logger = opts.logger || defaultLogger; From f375765cb4c07d4740df99a79254ae458beaa7d5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 15:02:58 +0100 Subject: [PATCH 170/232] update lockfile --- pnpm-lock.yaml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d9c3750ff..b9608ecb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,9 +106,9 @@ importers: integration-tests/cli/repo: dependencies: - '@openfn/language-common_1.11.0': - specifier: npm:@openfn/language-common@^1.11.0 - version: /@openfn/language-common@1.11.0 + '@openfn/language-common_1.11.1': + specifier: npm:@openfn/language-common@^1.11.1 + version: /@openfn/language-common@1.11.1 integration-tests/worker: dependencies: @@ -127,6 +127,9 @@ importers: '@types/node': specifier: ^18.15.13 version: 18.15.13 + '@types/rimraf': + specifier: ^3.0.2 + version: 3.0.2 ava: specifier: 5.3.1 version: 5.3.1 @@ -145,10 +148,12 @@ importers: typescript: specifier: ^5.1.6 version: 5.1.6 - devDependencies: - '@types/rimraf': - specifier: ^3.0.2 - version: 3.0.2 + + integration-tests/worker/tmp/repo: + dependencies: + '@openfn/language-common_latest': + specifier: npm:@openfn/language-common@^1.11.1 + version: /@openfn/language-common@1.11.1 packages/cli: dependencies: @@ -428,6 +433,10 @@ importers: specifier: ^5.1.6 version: 5.1.6 + packages/engine-multi/tmp/a/b/c: {} + + packages/engine-multi/tmp/repo: {} + packages/lightning-mock: dependencies: '@koa/router': @@ -1624,8 +1633,8 @@ packages: semver: 7.5.4 dev: true - /@openfn/language-common@1.11.0: - resolution: {integrity: sha512-fd7d2ML03qNTAKu1PY/9zCV/TL3ZVsf70CJSpJUW3uloazlj/fDLRPmwr4Qzb8aYUB0qSGh1xY16s/e7N3NhOQ==} + /@openfn/language-common@1.11.1: + resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==} dependencies: ajv: 8.12.0 axios: 1.1.3 @@ -1769,7 +1778,6 @@ packages: dependencies: '@types/minimatch': 5.1.2 '@types/node': 18.15.13 - dev: true /@types/gunzip-maybe@1.4.0: resolution: {integrity: sha512-dFP9GrYAR9KhsjTkWJ8q8Gsfql75YIKcg9DuQOj/IrlPzR7W+1zX+cclw1McV82UXAQ+Lpufvgk3e9bC8+HzgA==} @@ -1865,7 +1873,6 @@ packages: /@types/minimatch@5.1.2: resolution: {integrity: sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==} - dev: true /@types/minimist@1.2.2: resolution: {integrity: sha512-jhuKLIRrhvCPLqwPcx6INqmKeiA5EWrsCOPhrlFSrbrmU4ZMPjj5Ul/oLCMDO98XRUIwVm78xICz4EPCektzeQ==} @@ -1963,7 +1970,6 @@ packages: dependencies: '@types/glob': 8.1.0 '@types/node': 18.15.13 - dev: true /@types/scheduler@0.16.2: resolution: {integrity: sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==} From 3b7317b486ca2809da09e06407f3e282305a55f7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 15:07:30 +0100 Subject: [PATCH 171/232] bump node versions --- .circleci/config.yml | 2 +- .tool-versions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3ab52e894..ea09259be 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: cimg/node:18.12 + - image: cimg/node:18.18 resource_class: medium # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps diff --git a/.tool-versions b/.tool-versions index 5686ee0db..343dc534c 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -nodejs 18.12.1 +nodejs 18.18.2 From c6884685101b6e97f841a92dab6067cebe7bad57 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 15:08:31 +0100 Subject: [PATCH 172/232] lightning-mock: quick type fix --- packages/lightning-mock/src/api-sockets.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts index 06708ed9b..47b3e8cd0 100644 --- a/packages/lightning-mock/src/api-sockets.ts +++ b/packages/lightning-mock/src/api-sockets.ts @@ -135,7 +135,7 @@ const createSocketAPI = ( startAttempt, close: () => { server.close(); - wss.close(); + (wss as any).close(); }, }; From 13554c3e96a1a0b037c7ebd3bc20fb6fd3b470d0 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 15:15:07 +0100 Subject: [PATCH 173/232] engine: don't take an id --- packages/ws-worker/src/mock/runtime-engine.ts | 5 +---- packages/ws-worker/src/start.ts | 4 ++-- packages/ws-worker/test/mock/runtime-engine.test.ts | 10 ---------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 7b6a470cd..d598e13e0 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -51,9 +51,7 @@ export type WorkflowCompleteEvent = { error?: any; }; -let autoServerId = 0; - -function createMock(serverId?: string) { +function createMock() { const activeWorkflows = {} as Record; const bus = new EventEmitter(); const listeners: Record = {}; @@ -176,7 +174,6 @@ function createMock(serverId?: string) { }; return { - id: serverId || `${++autoServerId}`, on, once, execute, diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 2d8ebcc34..f74b3158c 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -74,10 +74,10 @@ if (args.lightning === 'mock') { } let engine; if (args.mock) { - engine = createMockRTE('rte'); + engine = createMockRTE(); logger.debug('Mock engine created'); } else { - engine = createRTE('rte', { repoDir: args.repoDir }); + engine = createRTE({ repoDir: args.repoDir }); logger.debug('engine created'); } diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index 3f933ea91..2d6739453 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -19,16 +19,6 @@ const sampleWorkflow = { ], } as ExecutionPlan; -test('mock runtime engine should have an id', (t) => { - const engine = create('22'); - const keys = Object.keys(engine); - t.assert(engine.id == '22'); - - // No need to test the full API, just make sure it smells right - t.assert(keys.includes('on')); - t.assert(keys.includes('execute')); -}); - test('getStatus() should should have no active workflows', (t) => { const engine = create(); const { active } = engine.getStatus(); From ff5b7e9f03e92d645c0f3cd2bcdda5aab3144e0d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 16:25:51 +0100 Subject: [PATCH 174/232] worker: better error handling --- packages/engine-multi/src/api/execute.ts | 14 +++++++++-- packages/engine-multi/src/api/lifecycle.ts | 10 ++++++++ .../engine-multi/src/test/worker-functions.js | 18 +++++++------ packages/engine-multi/src/worker/worker.ts | 1 + .../engine-multi/test/api/lifecycle.test.ts | 20 +++++++++++++++ packages/engine-multi/test/engine.test.ts | 25 +++++++++++++++++++ 6 files changed, 78 insertions(+), 10 deletions(-) diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 4aaefc64e..14b78c995 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -10,6 +10,7 @@ import { log, jobStart, jobComplete, + error, } from './lifecycle'; const execute = async (context: ExecutionContext) => { @@ -38,13 +39,22 @@ const execute = async (context: ExecutionContext) => { }; return callWorker('run', [state.plan, adaptorPaths], events).catch( (e: any) => { - // TODO what about errors then? + // TODO what information can I usefully provide here? + // DO I know which job I'm on? + // DO I know the thread id? + // Do I know where the error came from? + // console.log(' *** EXECUTE ERROR ***'); + // console.log(e); + + error(context, { workflowId: state.plan.id, error: e }); // If the worker file can't be found, we get: // code: MODULE_NOT_FOUND - // message: cannot find modulle (worker.js) + // message: cannot find module (worker.js) logger.error(e); + + // probbaly have to call complete write now and set the reason } ); }; diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index 327ab6260..ca13bba9f 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -115,3 +115,13 @@ export const log = ( ...event.message, }); }; + +export const error = (context: ExecutionContext, event: any) => { + const { threadId = '-', error } = event; + + context.emit(externalEvents.WORKFLOW_ERROR, { + threadId, + type: error.type || error.name || 'ERROR', + message: error.message || error.toString(), + }); +}; diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index 7ba254136..764bdbfcc 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -34,14 +34,16 @@ workerpool.worker({ threadId, }); } catch (err) { - console.error(err); - // @ts-ignore TODO sort out error typing - workerpool.workerEmit({ - type: 'worker:workflow-error', - workflowId, - message: err.message, - threadId, - }); + // console.error(err); + // // @ts-ignore TODO sort out error typing + // workerpool.workerEmit({ + // type: 'worker:workflow-error', + // workflowId, + // message: err.message, + // threadId, + // }); + // actually, just throw the error back out + throw err; } }, }); diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 3ffb7c016..f574167b2 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -31,6 +31,7 @@ workerpool.worker({ jobLogger, linker: { modules: adaptorPaths, + whitelist: [/^@openfn/], }, callbacks: { notify: (name: NotifyEvents, payload: any) => { diff --git a/packages/engine-multi/test/api/lifecycle.test.ts b/packages/engine-multi/test/api/lifecycle.test.ts index 9b692d0f5..853523668 100644 --- a/packages/engine-multi/test/api/lifecycle.test.ts +++ b/packages/engine-multi/test/api/lifecycle.test.ts @@ -8,6 +8,7 @@ import { workflowStart, jobStart, jobComplete, + error, } from '../../src/api/lifecycle'; import { WorkflowState } from '../../src/types'; import ExecutionContext from '../../src/classes/ExecutionContext'; @@ -185,3 +186,22 @@ test(`log: emits ${e.WORKFLOW_LOG}`, (t) => { log(context, event); }); }); + +// TODO not a very thorough test, still not really sure what I'm doing here +test(`error: emits ${e.WORKFLOW_ERROR}`, (t) => { + return new Promise((done) => { + const workflowId = 'a'; + + const context = createContext(workflowId); + context.on(e.WORKFLOW_ERROR, (evt) => { + t.is(evt.message, 'test'); + t.is(evt.workflowId, 'a'); + + done(); + }); + + const err = new Error('test'); + + error(context, { error: err }); + }); +}); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index dfd12bd4d..519670c6c 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -177,3 +177,28 @@ test('call listen before execute', (t) => { engine.execute(plan); }); }); + +test.only('catch and emit errors', (t) => { + return new Promise((done) => { + const p = path.resolve('src/test/worker-functions.js'); + const engine = createEngine(options, p); + + const plan = { + id: 'a', + jobs: [ + { + expression: 'throw new Error("test")', + }, + ], + }; + + engine.execute(plan); + + engine.listen(plan.id, { + [e.WORKFLOW_ERROR]: ({ message }) => { + t.is(message, 'test'); + done(); + }, + }); + }); +}); From 6705d714d91b43129495efd811df71f27cea2eea Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 16:49:55 +0100 Subject: [PATCH 175/232] worker: map error event to complete --- packages/ws-worker/src/api/execute.ts | 20 ++++++++++++++++++- packages/ws-worker/src/mock/runtime-engine.ts | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 3429d29b3..57acfe280 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -116,7 +116,9 @@ export function execute( addEvent('job-complete', onJobComplete), addEvent('log', onJobLog), // This will also resolve the promise - addEvent('workflow-complete', onWorkflowComplete) + addEvent('workflow-complete', onWorkflowComplete), + + addEvent('workflow-error', onWorkflowError) // TODO send autoinstall logs // are these associated with a workflow...? @@ -217,11 +219,27 @@ export async function onWorkflowComplete( await sendEvent(channel, ATTEMPT_COMPLETE, { final_dataclip_id: state.lastDataclipId!, status: 'success', // TODO + reason: 'ok', // Also TODO }); onComplete(result); } +// On errorr, for now, we just post to workflow complete +// No unit tests on this (not least because I think it'll change soon) +export async function onWorkflowError( + { state, channel, onComplete }: Context, + event: WorkflowErrorEvent +) { + await sendEvent(channel, ATTEMPT_COMPLETE, { + reason: 'fail', // TODO + final_dataclip_id: state.lastDataclipId!, + message: event.message, + }); + + onComplete({}); +} + export function onJobLog({ channel, state }: Context, event: JSONLog) { // lightning-friendly log object const log: ATTEMPT_LOG_PAYLOAD = { diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index d598e13e0..5c8e7f4db 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -48,7 +48,12 @@ export type WorkflowStartEvent = { export type WorkflowCompleteEvent = { workflowId: string; - error?: any; + error?: any; // hmm maybe not +}; + +export type WorkflowErrorEvent = { + workflowId: string; + message: string; }; function createMock() { From ee0830a142d7e8ba75cbfaf5632c00f78a8a3946 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 16:50:25 +0100 Subject: [PATCH 176/232] worker-tests: handle an error --- .../worker/test/integration.test.ts | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 0d2d2bf56..d38300242 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -64,7 +64,7 @@ test('should join attempts queue channel', (t) => { }); }); -test('should run a simple job with no compilation', (t) => { +test('should run a simple job with no compilation or adaptor', (t) => { return new Promise((done) => { initLightning(); lightning.on('attempt:complete', (evt) => { @@ -180,7 +180,7 @@ test('run a job which does NOT autoinstall common', (t) => { }); }); -test.only('run a job with initial state', (t) => { +test('run a job with initial state', (t) => { return new Promise((done) => { const attempt = { id: crypto.randomUUID(), @@ -219,6 +219,36 @@ test.only('run a job with initial state', (t) => { }); }); +test('blacklist a non-openfn adaptor', (t) => { + return new Promise((done) => { + const attempt = { + id: crypto.randomUUID(), + jobs: [ + { + adaptor: 'lodash@latest', + body: 'import _ from "lodash"', + }, + ], + }; + + initLightning(); + + // At the moment the error comes back to on complete + lightning.on('attempt:complete', (event) => { + const { payload } = event; + t.is(payload.reason, 'fail'); + t.is(payload.message, 'Error: module blacklisted: lodash'); + done(); + }); + + initWorker(); + + lightning.enqueueAttempt(attempt); + }); +}); + +test.todo('return some kind of error on compilation error'); + // test('run a job with complex behaviours (initial state, branching)', (t) => { // const attempt = { // id: 'a1', From 808d6b6886e543267ac7b19eac959f5ea857fba8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 16:57:15 +0100 Subject: [PATCH 177/232] trypings --- packages/ws-worker/src/api/execute.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 57acfe280..9adc66985 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -21,6 +21,7 @@ import type { ExecutionPlan } from '@openfn/runtime'; import type { JSONLog, Logger } from '@openfn/logger'; import { WorkflowCompleteEvent, + WorkflowErrorEvent, WorkflowStartEvent, } from '../mock/runtime-engine'; From 9b16a8d8021fc7b3fff8afbb63e4a5ff771ac527 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 18:28:18 +0100 Subject: [PATCH 178/232] engine: preload credentials instead of loading on demand --- packages/engine-multi/src/api/execute.ts | 12 +++- .../src/api/preload-credentials.ts | 22 ++++++ packages/engine-multi/src/worker/worker.ts | 4 ++ .../test/api/preload-credentials.test.ts | 68 +++++++++++++++++++ .../engine-multi/test/integration.test.ts | 36 ++++++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 packages/engine-multi/src/api/preload-credentials.ts create mode 100644 packages/engine-multi/test/api/preload-credentials.test.ts diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 14b78c995..5ddb40d13 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -12,12 +12,22 @@ import { jobComplete, error, } from './lifecycle'; +import preloadCredentials from './preload-credentials'; const execute = async (context: ExecutionContext) => { - const { state, callWorker, logger } = context; + const { state, callWorker, logger, options } = context; const adaptorPaths = await autoinstall(context); await compile(context); + + // unfortunately we have to preload all credentials + // I don't know any way to send data back into the worker once started + // there is a shared memory thing but I'm not sure how it works yet + // and not convinced we can use it for two way communication + if (options.resolvers?.credentials) { + await preloadCredentials(state.plan as any, options.resolvers?.credentials); + } + const events = { [workerEvents.WORKFLOW_START]: (evt: workerEvents.WorkflowStartEvent) => { workflowStart(context, evt); diff --git a/packages/engine-multi/src/api/preload-credentials.ts b/packages/engine-multi/src/api/preload-credentials.ts new file mode 100644 index 000000000..fb9545ff7 --- /dev/null +++ b/packages/engine-multi/src/api/preload-credentials.ts @@ -0,0 +1,22 @@ +import { CompiledExecutionPlan } from '@openfn/runtime'; + +export default async ( + plan: CompiledExecutionPlan, + loader: (id: string) => Promise +) => { + const loaders: Promise[] = []; + + Object.values(plan.jobs).forEach((job) => { + if (typeof job.configuration === 'string') { + loaders.push( + new Promise(async (resolve) => { + job.configuration = await loader(job.configuration as string); + resolve(); + }) + ); + } + }); + + await Promise.all(loaders); + return plan; +}; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index f574167b2..cef1cafb7 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -34,6 +34,10 @@ workerpool.worker({ whitelist: [/^@openfn/], }, callbacks: { + // TODO: this won't actually work across the worker boundary + // For now I am preloading credentials + // resolveCredential: async (id: string) => { + // }, notify: (name: NotifyEvents, payload: any) => { // convert runtime notify events to internal engine events publish(plan.id!, `worker:${name}`, payload); diff --git a/packages/engine-multi/test/api/preload-credentials.test.ts b/packages/engine-multi/test/api/preload-credentials.test.ts new file mode 100644 index 000000000..e31c04191 --- /dev/null +++ b/packages/engine-multi/test/api/preload-credentials.test.ts @@ -0,0 +1,68 @@ +import test from 'ava'; +import preloadCredentials from '../../src/api/preload-credentials'; +import { CompiledExecutionPlan } from '@openfn/runtime'; + +// Not very good test coverage +test('handle a plan with no credentials', async (t) => { + let timesCalled = 0; + + const loader = async (id: string) => { + timesCalled++; + return `loaded-${id}`; + }; + + const plan = { + id: 'a', + jobs: [ + { + expression: '.', + }, + { + expression: '.', + }, + { + expression: '.', + }, + ], + } as unknown as CompiledExecutionPlan; + + const planCopy = JSON.parse(JSON.stringify(plan)); + const result = await preloadCredentials(plan, loader); + + t.is(timesCalled, 0); + t.deepEqual(planCopy, result); +}); + +test('handle a plan with credentials', async (t) => { + let timesCalled = 0; + + const loader = async (id: string) => { + timesCalled++; + return `loaded-${id}`; + }; + + const plan = { + id: 'a', + jobs: [ + { + expression: '.', + configuration: 'a', + }, + { + expression: '.', + configuration: 'b', + }, + { + expression: '.', + configuration: 'c', + }, + ], + } as unknown as CompiledExecutionPlan; + + const result = await preloadCredentials(plan, loader); + + t.is(timesCalled, 3); + t.is(plan.jobs[0].configuration, 'loaded-a'); + t.is(plan.jobs[1].configuration, 'loaded-b'); + t.is(plan.jobs[2].configuration, 'loaded-c'); +}); diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index c24d27845..7a4a5a308 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -187,5 +187,41 @@ test('evaluate conditional edges', (t) => { }); }); +test('preload credentials', (t) => { + return new Promise((done) => { + let didCallLoader = true; + + const loader = (id: string) => + new Promise((resolve) => { + setTimeout(() => { + didCallLoader = true; + t.is(id, 'secret'); + resolve({}); + }, 100); + }); + + const api = createAPI({ + logger, + resolvers: { + credentials: loader, + }, + }); + + const jobs = [ + { + id: 'a', + configuration: 'secret', + }, + ]; + + const plan = createPlan(jobs); + + api.execute(plan).on('workflow-complete', ({ state }) => { + t.true(didCallLoader); + done(); + }); + }); +}); + test.todo('should report an error'); test.todo('various workflow options (start, initial state)'); From 466ff01adc8810f901d0557038d5dd1c7188331e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 18:49:42 +0100 Subject: [PATCH 179/232] types --- packages/engine-multi/src/engine.ts | 10 +++--- packages/engine-multi/src/index.ts | 2 ++ .../engine-multi/src/{types.d.ts => types.ts} | 36 ++++++++----------- 3 files changed, 22 insertions(+), 26 deletions(-) rename packages/engine-multi/src/{types.d.ts => types.ts} (70%) diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 9f0373374..5f77ecff7 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -119,9 +119,8 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { const getWorkflowStatus = (workflowId: string) => states[workflowId]?.status; - // TODO are we totally sure this takes a standard xplan? - // Well, it MUST have an ID or there's trouble - const executeWrapper = (plan: ExecutionPlan) => { + // TODO maybe engine options is too broad? + const executeWrapper = (plan: ExecutionPlan, opts: EngineOptions) => { options.logger!.debug('executing plan ', plan?.id ?? ''); const workflowId = plan.id!; // TODO throw if plan is invalid @@ -133,7 +132,10 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { state, logger: options.logger!, callWorker: engine.callWorker, - options, + options: { + ...options, + ...opts, + }, }); contexts[workflowId] = createWorkflowEvents(engine, context, workflowId); diff --git a/packages/engine-multi/src/index.ts b/packages/engine-multi/src/index.ts index 3bfff051f..ffc5bf360 100644 --- a/packages/engine-multi/src/index.ts +++ b/packages/engine-multi/src/index.ts @@ -1,3 +1,5 @@ import createEngine from './api'; export default createEngine; + +export * from './types'; diff --git a/packages/engine-multi/src/types.d.ts b/packages/engine-multi/src/types.ts similarity index 70% rename from packages/engine-multi/src/types.d.ts rename to packages/engine-multi/src/types.ts index ec611e031..ded4950fa 100644 --- a/packages/engine-multi/src/types.d.ts +++ b/packages/engine-multi/src/types.ts @@ -1,28 +1,21 @@ // ok first of allI want to capture the key interfaces -import { JSONLog, Logger } from '@openfn/logger'; +import { Logger } from '@openfn/logger'; import { ExecutionPlan } from '@openfn/runtime'; import type { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; import { RTEOptions } from './api'; import { ExternalEvents, EventMap } from './events'; +import { EngineOptions } from './engine'; -// TODO hmm, not sure about this - event handler for what? -export type EventHandler = ( - event: EventPayloadLookup[T] -) => void; +export type Resolver = (id: string) => Promise; -type Resolver = (id: string) => Promise; - -type Resolvers = { +export type Resolvers = { credential?: Resolver; - state?: Resolver; + state?: Resolver; }; -type ExecuteOptions = { - sanitize: any; // log sanitise options - noCompile: any; // skip compilation (useful in test) -}; +export type EventHandler = (event: any) => void; export type WorkflowState = { id: string; @@ -37,11 +30,11 @@ export type WorkflowState = { options: any; // TODO this is wf specific options, like logging policy }; -export type CallWorker = ( +export type CallWorker = ( task: string, - args: any[] = [], - events: any = {} -) => workerpool.Promise; + args: any[], + events: any +) => workerpool.Promise; export type ExecutionContextConstructor = { state: WorkflowState; @@ -51,7 +44,7 @@ export type ExecutionContextConstructor = { }; export interface ExecutionContext extends EventEmitter { - constructor(args: ExecutionContextConstructor); + constructor(args: ExecutionContextConstructor): ExecutionContext; options: RTEOptions; // TODO maybe. bring them in here? state: WorkflowState; logger: Logger; @@ -67,7 +60,7 @@ export interface EngineAPI extends EventEmitter { callWorker: CallWorker; } -interface RuntimeEngine extends EventEmitter { +export interface RuntimeEngine extends EventEmitter { //id: string // human readable instance id // actually I think the id is on the worker, not the engine @@ -78,9 +71,8 @@ interface RuntimeEngine extends EventEmitter { // Kinda convenient but not actually needed execute( plan: ExecutionPlan, - resolvers: Resolvers, - options: ExecuteOptions = {} - ); + options: EngineOptions + ): Pick; // TODO my want some maintenance APIs, like getStatus. idk } From 17ebba62e0a0f65e111259ac2165b7805fa408d3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 18:54:53 +0100 Subject: [PATCH 180/232] worker-tests: add a sort of working credential test --- integration-tests/worker/package.json | 4 +- .../worker/test/integration.test.ts | 73 +++++++++++++++++-- packages/engine-multi/src/api/execute.ts | 4 - packages/ws-worker/src/api/execute.ts | 7 +- pnpm-lock.yaml | 6 ++ 5 files changed, 81 insertions(+), 13 deletions(-) diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index ce5e6c12b..177c07c80 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -11,7 +11,8 @@ "build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/worker/dist --no-version", "build": "pnpm build:pack && docker build --tag worker-integration-tests .", "start": "docker run worker-integration-tests", - "test": "pnpm clean && npx ava -s --timeout 2m" + "_test": "pnpm clean && npx ava -s --timeout 2m", + "test": "npx ava -s --timeout 2m" }, "dependencies": { "@openfn/engine-multi": "workspace:^", @@ -22,6 +23,7 @@ "@types/rimraf": "^3.0.2", "ava": "5.3.1", "date-fns": "^2.30.0", + "koa": "^2.13.4", "rimraf": "^3.0.2", "ts-node": "10.8.1", "tslib": "^2.4.0", diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index d38300242..54ec4759f 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -1,6 +1,7 @@ import test from 'ava'; import path from 'node:path'; import crypto from 'node:crypto'; +import Koa from 'koa'; import createLightningServer from '@openfn/lightning-mock'; @@ -219,6 +220,73 @@ test('run a job with initial state', (t) => { }); }); +// TODO this sort of works but the server side of it does not +// Will work on it more +test('run a job with credentials', (t) => { + // Set up a little web server to receive a request + // (there are easier ways to do this, but this is an INTEGRATION test right??) + const PORT = 4826; + const createServer = () => { + const app = new Koa(); + + app.use(async (ctx, next) => { + console.log('GET!'); + // TODO check basic credential + ctx.body = '{ message: "ok" }'; + ctx.response.headers['Content-Type'] = 'application/json'; + ctx.response.status = 200; + }); + + return app.listen(PORT); + }; + + return new Promise((done) => { + const server = createServer(); + const config = { + username: 'logan', + password: 'jeangr3y', + }; + + const attempt = { + id: crypto.randomUUID(), + jobs: [ + { + adaptor: '@openfn/language-http@latest', + body: `fn((s) => { + console.log(s); + return s + })`, + // body: `get("http://localhost:${PORT}") + // fn((s) => { + // console.log(s); + // return s; + // })`, + credential: 'c', + }, + ], + }; + + initLightning(); + + lightning.addCredential('c', config); + + lightning.on('attempt:complete', () => { + try { + const result = lightning.getResult(attempt.id); + t.deepEqual(result.configuration, config); + + server.close(); + } catch (e) { + console.log(e); + } + done(); + }); + + initWorker(); + lightning.enqueueAttempt(attempt); + }); +}); + test('blacklist a non-openfn adaptor', (t) => { return new Promise((done) => { const attempt = { @@ -283,8 +351,3 @@ test.todo('return some kind of error on compilation error'); // }); // }); // }); - -// maybe create a http server with basic auth on the endpoint -// use http adaptor to call the server -// obviously credential is lazy loaded -test.todo('run a job which requires credentials'); diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 5ddb40d13..646b562b1 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -53,8 +53,6 @@ const execute = async (context: ExecutionContext) => { // DO I know which job I'm on? // DO I know the thread id? // Do I know where the error came from? - // console.log(' *** EXECUTE ERROR ***'); - // console.log(e); error(context, { workflowId: state.plan.id, error: e }); @@ -63,8 +61,6 @@ const execute = async (context: ExecutionContext) => { // message: cannot find module (worker.js) logger.error(e); - - // probbaly have to call complete write now and set the reason } ); }; diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 9adc66985..e97b98598 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -24,6 +24,7 @@ import { WorkflowErrorEvent, WorkflowStartEvent, } from '../mock/runtime-engine'; +import type { RuntimeEngine } from '@openfn/engine-multi'; const enc = new TextDecoder('utf-8'); @@ -57,7 +58,7 @@ const eventMap = { // this thing will do all the work export function execute( channel: Channel, - engine: any, // TODO typing! + engine: RuntimeEngine, logger: Logger, // TODO first thing we'll do here is pull the plan plan: ExecutionPlan @@ -130,7 +131,7 @@ export function execute( engine.listen(plan.id, listeners); const resolvers = { - credential: (id: string) => loadCredential(channel, id), + credentials: (id: string) => loadCredential(channel, id), // TODO not supported right now // dataclip: (id: string) => loadDataclip(channel, id), @@ -144,7 +145,7 @@ export function execute( logger.success('dataclip loaded'); logger.debug(plan.initialState); } - engine.execute(plan, resolvers); + engine.execute(plan, { resolvers }); }); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9608ecb3..751c7c54b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,6 +136,9 @@ importers: date-fns: specifier: ^2.30.0 version: 2.30.0 + koa: + specifier: ^2.13.4 + version: 2.13.4 rimraf: specifier: ^3.0.2 version: 3.0.2 @@ -154,6 +157,9 @@ importers: '@openfn/language-common_latest': specifier: npm:@openfn/language-common@^1.11.1 version: /@openfn/language-common@1.11.1 + lodash_latest: + specifier: npm:lodash@^4.17.21 + version: /lodash@4.17.21 packages/cli: dependencies: From 13e824d9313fab93db06587d4c5acdb2a00eb09e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 18 Oct 2023 18:59:47 +0100 Subject: [PATCH 181/232] types --- packages/engine-multi/src/engine.ts | 7 ++++++- packages/engine-multi/src/types.ts | 2 +- packages/ws-worker/src/api/execute.ts | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 5f77ecff7..1a1cf754a 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -119,8 +119,13 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { const getWorkflowStatus = (workflowId: string) => states[workflowId]?.status; + // TODO too much logic in this execute function, needs farming out + // I don't mind having a wrapper here but it must be super thin // TODO maybe engine options is too broad? - const executeWrapper = (plan: ExecutionPlan, opts: EngineOptions) => { + const executeWrapper = ( + plan: ExecutionPlan, + opts: Partial + ) => { options.logger!.debug('executing plan ', plan?.id ?? ''); const workflowId = plan.id!; // TODO throw if plan is invalid diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index ded4950fa..7e83c1a40 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -71,7 +71,7 @@ export interface RuntimeEngine extends EventEmitter { // Kinda convenient but not actually needed execute( plan: ExecutionPlan, - options: EngineOptions + options?: Partial ): Pick; // TODO my want some maintenance APIs, like getStatus. idk diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index e97b98598..a91d6b225 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -128,7 +128,7 @@ export function execute( // Or is this just a log? // Or a generic metric? ); - engine.listen(plan.id, listeners); + engine.listen(plan.id!, listeners); const resolvers = { credentials: (id: string) => loadCredential(channel, id), From 8e9251dd0511cdcfb410842cefdada56912cdfc6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 14:32:22 +0100 Subject: [PATCH 182/232] runtime: support disabled edges --- packages/runtime/src/execute/job.ts | 13 +++++++----- packages/runtime/src/types.ts | 2 ++ packages/runtime/test/execute/plan.test.ts | 23 ++++++++++++++++++++++ 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts index dafddbae1..de299ab37 100644 --- a/packages/runtime/src/execute/job.ts +++ b/packages/runtime/src/execute/job.ts @@ -93,12 +93,15 @@ const executeJob = async ( if (job.next) { for (const nextJobId in job.next) { const edge = job.next[nextJobId]; - if ( - edge && - (edge === true || !edge.condition || edge.condition(result)) - ) { - next.push(nextJobId); + if (!edge) { + continue; } + if (typeof edge == 'object') { + if (edge.disabled || !edge.condition || !edge.condition(result)) { + continue; + } + } + next.push(nextJobId); // TODO errors } } diff --git a/packages/runtime/src/types.ts b/packages/runtime/src/types.ts index 41cba641c..61a8fe19b 100644 --- a/packages/runtime/src/types.ts +++ b/packages/runtime/src/types.ts @@ -76,6 +76,7 @@ export type JobEdge = | { condition?: string; // Javascript expression (function body, not function) label?: string; + disabled?: boolean; }; export type JobNodeID = string; @@ -84,6 +85,7 @@ export type CompiledJobEdge = | boolean | { condition?: Function; + disabled?: boolean; }; export type CompiledJobNode = Omit & { diff --git a/packages/runtime/test/execute/plan.test.ts b/packages/runtime/test/execute/plan.test.ts index 2696eb3cd..a9459c15a 100644 --- a/packages/runtime/test/execute/plan.test.ts +++ b/packages/runtime/test/execute/plan.test.ts @@ -462,6 +462,29 @@ test('skip edge based on state in the condition ', async (t) => { t.is(result.data?.x, 10); }); +test('do not traverse a disabled edge', async (t) => { + const plan: ExecutionPlan = { + jobs: [ + { + id: 'job1', + expression: 'export default [(s) => { s.data.x = 10; return s;}]', + next: { + job2: { + disabled: true, + condition: 'true', + }, + }, + }, + { + id: 'job2', + expression: 'export default [() => ({ data: { x: 20 } })]', + }, + ], + }; + const result = await execute(plan, {}, mockLogger); + t.is(result.data?.x, 10); +}); + test('execute a two-job execution plan', async (t) => { const plan: ExecutionPlan = { initialState: { data: { x: 0 } }, From 60750653e0bc9374f87f370c468629097a574621 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 14:44:11 +0100 Subject: [PATCH 183/232] worker: handle incoming enabled flag --- packages/ws-worker/src/types.d.ts | 1 + .../ws-worker/src/util/convert-attempt.ts | 29 ++++++++++++++----- .../test/util/convert-attempt.test.ts | 18 ++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index ea63ce151..d93de5497 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -34,6 +34,7 @@ export interface Edge { condition?: string; error_path?: boolean; errors?: any; + enabled?: boolean; } // An attempt object returned by Lightning diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index 46d26e627..f6e4e3142 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -1,5 +1,10 @@ import crypto from 'node:crypto'; -import type { ExecutionPlan, JobNode, JobNodeID } from '@openfn/runtime'; +import type { + ExecutionPlan, + JobNode, + JobNodeID, + JobEdge, +} from '@openfn/runtime'; import { Attempt } from '../types'; export default (attempt: Attempt): ExecutionPlan => { @@ -35,8 +40,10 @@ export default (attempt: Attempt): ExecutionPlan => { const connectedEdges = edges.filter((e) => e.source_trigger_id === id); if (connectedEdges.length) { nodes[id].next = connectedEdges.reduce((obj, edge) => { - // @ts-ignore - obj[edge.target_job_id] = true; + if (!edge.disabled) { + // @ts-ignore + obj[edge.target_job_id] = true; + } return obj; }, {}); } else { @@ -51,7 +58,7 @@ export default (attempt: Attempt): ExecutionPlan => { nodes[id] = { id, - configuration: job.credential, // TODO runtime needs to support string credentials + configuration: job.credential, expression: job.body, adaptor: job.adaptor, }; @@ -64,12 +71,18 @@ export default (attempt: Attempt): ExecutionPlan => { const next = edges .filter((e) => e.source_job_id === id) .reduce((obj, edge) => { - // @ts-ignore - obj[edge.target_job_id] = edge.condition - ? { expression: edge.condition } + const newEdge: JobEdge = {}; + if (edge.condition) { + newEdge.expression = edge.condition; + } + if (edge.enabled === false) { + newEdge.disabled = true; + } + obj[edge.target_job_id] = Object.keys(newEdge).length + ? newEdge : true; return obj; - }, {}); + }, {} as Record); if (Object.keys(next).length) { nodes[id].next = next; diff --git a/packages/ws-worker/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts index 7518b2c32..1977fcffa 100644 --- a/packages/ws-worker/test/util/convert-attempt.test.ts +++ b/packages/ws-worker/test/util/convert-attempt.test.ts @@ -265,3 +265,21 @@ test('convert two linked jobs with an edge condition', (t) => { ], }); }); + +test('convert two linked jobs with a disabled edge', (t) => { + const attempt: Partial = { + id: 'w', + jobs: [createNode({ id: 'a' }), createNode({ id: 'b' })], + triggers: [], + edges: [createEdge('a', 'b', { enabled: false })], + }; + const result = convertAttempt(attempt as Attempt); + + t.deepEqual(result, { + id: 'w', + jobs: [ + createJob({ id: 'a', next: { b: { disabled: true } } }), + createJob({ id: 'b' }), + ], + }); +}); From ed34d0448294e947ed5d831140237e71a076fe39 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 14:48:17 +0100 Subject: [PATCH 184/232] engine: resolveCredential should be singular --- packages/engine-multi/src/api.ts | 2 +- packages/engine-multi/src/api/execute.ts | 4 ++-- packages/engine-multi/test/integration.test.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index bf6408efc..0928a5b86 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -13,7 +13,7 @@ type Resolver = (id: string) => Promise; // A list of helper functions which basically resolve ids into JSON // to lazy load assets export type LazyResolvers = { - credentials?: Resolver; + credential?: Resolver; state?: Resolver; expressions?: Resolver; }; diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 646b562b1..6539d0385 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -24,8 +24,8 @@ const execute = async (context: ExecutionContext) => { // I don't know any way to send data back into the worker once started // there is a shared memory thing but I'm not sure how it works yet // and not convinced we can use it for two way communication - if (options.resolvers?.credentials) { - await preloadCredentials(state.plan as any, options.resolvers?.credentials); + if (options.resolvers?.credential) { + await preloadCredentials(state.plan as any, options.resolvers?.credential); } const events = { diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 7a4a5a308..5d78199d3 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -203,7 +203,7 @@ test('preload credentials', (t) => { const api = createAPI({ logger, resolvers: { - credentials: loader, + credential: loader, }, }); From 3f62a4be5a57d86891dafa0e331f688cde876b07 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 15:01:39 +0100 Subject: [PATCH 185/232] various: fixing resolver types and values --- packages/ws-worker/src/api/execute.ts | 6 ++-- packages/ws-worker/src/mock/resolvers.ts | 4 +-- packages/ws-worker/src/mock/runtime-engine.ts | 24 +++------------- .../ws-worker/src/util/convert-attempt.ts | 2 +- packages/ws-worker/test/api/execute.test.ts | 6 ++-- .../test/mock/runtime-engine.test.ts | 4 +-- .../test/util/convert-attempt.test.ts | 28 +++++++++++++++++++ 7 files changed, 43 insertions(+), 31 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index a91d6b225..0989fde03 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -24,7 +24,7 @@ import { WorkflowErrorEvent, WorkflowStartEvent, } from '../mock/runtime-engine'; -import type { RuntimeEngine } from '@openfn/engine-multi'; +import type { RuntimeEngine, Resolvers } from '@openfn/engine-multi'; const enc = new TextDecoder('utf-8'); @@ -131,11 +131,11 @@ export function execute( engine.listen(plan.id!, listeners); const resolvers = { - credentials: (id: string) => loadCredential(channel, id), + credential: (id: string) => loadCredential(channel, id), // TODO not supported right now // dataclip: (id: string) => loadDataclip(channel, id), - }; + } as Resolvers; // TODO we nede to remove this from here nad let the runtime take care of it through // the resolver. See https://github.com/OpenFn/kit/issues/403 diff --git a/packages/ws-worker/src/mock/resolvers.ts b/packages/ws-worker/src/mock/resolvers.ts index 871c88f14..489107e95 100644 --- a/packages/ws-worker/src/mock/resolvers.ts +++ b/packages/ws-worker/src/mock/resolvers.ts @@ -1,5 +1,5 @@ import type { State, Credential } from '../types'; -import { LazyResolvers } from './runtime-engine'; +import { Resolvers } from '@openfn/engine-multi'; const mockResolveCredential = (_credId: string) => new Promise((resolve) => @@ -23,4 +23,4 @@ export default { credentials: mockResolveCredential, state: mockResolveState, expressions: mockResolveExpressions, -} as LazyResolvers; +} as Resolvers; diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 5c8e7f4db..e7a984de9 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -1,26 +1,10 @@ import crypto from 'node:crypto'; import { EventEmitter } from 'node:events'; import type { ExecutionPlan, JobNode } from '@openfn/runtime'; - -import type { State, Credential } from '../types'; +import type { Resolvers } from '@openfn/engine-multi'; +import type { State } from '../types'; import mockResolvers from './resolvers'; -// A mock runtime engine - -// Runs ExecutionPlans(XPlans) in worker threads -// May need to lazy-load resources -// The mock engine will return expression JSON as state - -type Resolver = (id: string) => Promise; - -// A list of helper functions which basically resolve ids into JSON -// to lazy load assets -export type LazyResolvers = { - credential?: Resolver; - state?: Resolver; - expressions?: Resolver; -}; - export type EngineEvent = | 'job-start' | 'job-complete' @@ -88,7 +72,7 @@ function createMock() { workflowId: string, job: JobNode, initialState = {}, - resolvers: LazyResolvers = mockResolvers + resolvers: Resolvers = mockResolvers ) => { const { id, expression, configuration, adaptor } = job; @@ -143,7 +127,7 @@ function createMock() { // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = ( xplan: ExecutionPlan, - resolvers: LazyResolvers = mockResolvers + { resolvers }: { resolvers: Resolvers } = mockResolvers ) => { const { id, jobs, initialState } = xplan; const workflowId = id; diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index f6e4e3142..f7b29130a 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -40,7 +40,7 @@ export default (attempt: Attempt): ExecutionPlan => { const connectedEdges = edges.filter((e) => e.source_trigger_id === id); if (connectedEdges.length) { nodes[id].next = connectedEdges.reduce((obj, edge) => { - if (!edge.disabled) { + if (edge.enabled !== false) { // @ts-ignore obj[edge.target_job_id] = true; } diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 2b69c31f5..8ea57fe16 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -337,7 +337,7 @@ test('execute should return the final result', async (t) => { }); // TODO this is more of an engine test really, but worth having I suppose -test('execute should lazy-load a credential', async (t) => { +test.only('execute should lazy-load a credential', async (t) => { const logger = createMockLogger(); let didCallCredentials = false; @@ -349,7 +349,7 @@ test('execute should lazy-load a credential', async (t) => { return {}; }, }); - const engine = createMockRTE('rte'); + const engine = createMockRTE(); const plan = { id: 'a', @@ -378,7 +378,7 @@ test('execute should lazy-load initial state', async (t) => { return toArrayBuffer({}); }, }); - const engine = createMockRTE('rte'); + const engine = createMockRTE(); const plan: Partial = { id: 'a', diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index 2d6739453..68413113e 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -145,9 +145,9 @@ test('resolve credential before job-start if credential is a string', async (t) return {}; }; - const engine = create('1'); + const engine = create(); // @ts-ignore - engine.execute(wf, { credential }); + engine.execute(wf, { resolvers: { credential } }); await waitForEvent(engine, 'job-start'); t.true(didCallCredentials); diff --git a/packages/ws-worker/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts index 1977fcffa..6d115e709 100644 --- a/packages/ws-worker/test/util/convert-attempt.test.ts +++ b/packages/ws-worker/test/util/convert-attempt.test.ts @@ -208,6 +208,34 @@ test('convert a single trigger with two edges', (t) => { }); }); +test('convert a disabled trigger', (t) => { + const attempt: Partial = { + id: 'w', + triggers: [createTrigger()], + jobs: [createNode({ id: 'a' })], + edges: [ + { + id: 't-a', + source_trigger_id: 't', + target_job_id: 'a', + enabled: false, + }, + ], + }; + const result = convertAttempt(attempt as Attempt); + + t.deepEqual(result, { + id: 'w', + jobs: [ + { + id: 't', + next: {}, + }, + createJob({ id: 'a' }), + ], + }); +}); + test('convert two linked jobs', (t) => { const attempt: Partial = { id: 'w', From 525745a91b86803bb68f5e2202e8cd9e81d3070a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 15:05:07 +0100 Subject: [PATCH 186/232] worker: fix more typings --- packages/ws-worker/src/mock/runtime-engine.ts | 2 +- packages/ws-worker/src/util/convert-attempt.ts | 2 +- packages/ws-worker/test/util/convert-attempt.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index e7a984de9..501cc8fe6 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -127,7 +127,7 @@ function createMock() { // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = ( xplan: ExecutionPlan, - { resolvers }: { resolvers: Resolvers } = mockResolvers + { resolvers }: { resolvers?: Resolvers } = { resolvers: mockResolvers } ) => { const { id, jobs, initialState } = xplan; const workflowId = id; diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index f7b29130a..bb2b5570d 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -73,7 +73,7 @@ export default (attempt: Attempt): ExecutionPlan => { .reduce((obj, edge) => { const newEdge: JobEdge = {}; if (edge.condition) { - newEdge.expression = edge.condition; + newEdge.condition = edge.condition; } if (edge.enabled === false) { newEdge.disabled = true; diff --git a/packages/ws-worker/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts index 6d115e709..ea02d294b 100644 --- a/packages/ws-worker/test/util/convert-attempt.test.ts +++ b/packages/ws-worker/test/util/convert-attempt.test.ts @@ -288,7 +288,7 @@ test('convert two linked jobs with an edge condition', (t) => { t.deepEqual(result, { id: 'w', jobs: [ - createJob({ id: 'a', next: { b: { expression: condition } } }), + createJob({ id: 'a', next: { b: { condition } } }), createJob({ id: 'b' }), ], }); From 3f213142f8eef3421c0859395160ccf39b682d5b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 15:27:17 +0100 Subject: [PATCH 187/232] engine: lock down child process.env --- packages/engine-multi/src/api/call-worker.ts | 15 +++-- .../engine-multi/src/test/worker-functions.js | 6 ++ packages/engine-multi/src/types.ts | 2 +- .../engine-multi/test/api/call-worker.test.ts | 55 ++++++++++++++++++- 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 2d895428d..65743d79f 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -10,10 +10,14 @@ type WorkerEvent = { }; // Adds a `callWorker` function to the API object, which will execute a task in a worker -export default function initWorkers(api: EngineAPI, workerPath: string) { +export default function initWorkers( + api: EngineAPI, + workerPath: string, + env = {} +) { // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path - const workers = createWorkers(workerPath); + const workers = createWorkers(workerPath, env); api.callWorker = (task: string, args: any[] = [], events: any = {}) => workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { @@ -23,7 +27,7 @@ export default function initWorkers(api: EngineAPI, workerPath: string) { }); } -export function createWorkers(workerPath: string) { +export function createWorkers(workerPath: string, env: any) { let resolvedWorkerPath; if (workerPath) { // If a path to the worker has been passed in, just use it verbatim @@ -39,9 +43,8 @@ export function createWorkers(workerPath: string) { workerThreadOpts: { // Note that we have to pass this explicitly to run in ava's test runner execArgv: ['--no-warnings', '--experimental-vm-modules'], - // // TODO if this unset, can the thread read the parent env? - // Also todo I think this hides experimental vm modules so it all breaks - // env: {}, + // Important to override the child env so that it cannot access the parent env + env, }, }); } diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index 764bdbfcc..c95a39abb 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -14,6 +14,12 @@ workerpool.worker({ return result; }, + readEnv: (key) => { + if (key) { + return process.env[key]; + } + return process.env; + }, // very very simple intepretation of a run function // Most tests should use the mock-worker instead run: (plan, adaptorPaths) => { diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index 7e83c1a40..0b1895e43 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -33,7 +33,7 @@ export type WorkflowState = { export type CallWorker = ( task: string, args: any[], - events: any + events?: any ) => workerpool.Promise; export type ExecutionContextConstructor = { diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index bba65aa66..258af8923 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -6,17 +6,20 @@ import { EngineAPI } from '../../src/types'; let api = {} as EngineAPI; +const workerPath = path.resolve('src/test/worker-functions.js'); + test.before(() => { - const workerPath = path.resolve('src/test/worker-functions.js'); initWorkers(api, workerPath); }); +// TODO should I be tearing down the pool each time? + test('initWorkers should add a callWorker function', (t) => { t.assert(typeof api.callWorker === 'function'); }); test('callWorker should return the default result', async (t) => { - const result = await api.callWorker('test'); + const result = await api.callWorker('test', []); t.is(result, 42); }); @@ -62,3 +65,51 @@ test('callWorker should execute in a different process', async (t) => { api.callWorker('test', [], { message: onCallback }); }); }); + +test('If no env is passed, worker thread should be able to access parent env', async (t) => { + const badAPI = {} as EngineAPI; + initWorkers(badAPI, workerPath, null!); + + // Set up a special key on process.env + const code = '76ytghjs'; + process.env.TEST = code; + + // try and read that key inside the thread + const result = await badAPI.callWorker('readEnv', ['TEST']); + + // voila + t.is(result, code); +}); + +test('By default, worker thread cannot access parent env if env not set', async (t) => { + const defaultAPI = {} as EngineAPI; + initWorkers(defaultAPI, workerPath, undefined); + + // Set up a special key on process.env + const code = '76ytghjs'; + process.env.TEST = code; + + // try and read that key inside the thread + const result = await defaultAPI.callWorker('readEnv', ['TEST']); + + // No fish + t.is(result, undefined); +}); + +test('Worker thread cannot access parent env if custom env is passted', async (t) => { + const customAPI = {} as EngineAPI; + initWorkers(customAPI, workerPath, { NODE_ENV: 'production' }); + + // Set up a special key on process.env + const code = '76ytghjs'; + process.env.TEST = code; + + // try and read that key inside the thread + const result = await customAPI.callWorker('readEnv', ['TEST']); + + // No fish + t.is(result, undefined); + + const result2 = await customAPI.callWorker('readEnv', ['NODE_ENV']); + t.is(result2, 'production'); +}); From e4a4f86ecfde750dd5faf4804d375967bd021eff Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 15:30:48 +0100 Subject: [PATCH 188/232] workers: skip flaky test --- packages/engine-multi/test/worker/worker-pool.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index 4d6b5acd5..ca6d6c656 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -67,7 +67,8 @@ test.serial.skip('workers should not affect each other', async (t) => { t.is(result, undefined); }); -test.serial( +// maybe flaky? +test.serial.skip( 'workers should not affect each other if global scope is frozen', async (t) => { t.is(global.x, undefined); From f8517b8de6b450100d82d0739d86ac67f4d90582 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 15:33:10 +0100 Subject: [PATCH 189/232] worker: add API to close workers --- packages/engine-multi/src/api/call-worker.ts | 4 ++++ packages/engine-multi/src/types.ts | 1 + packages/engine-multi/test/api/call-worker.test.ts | 10 ++++++++++ 3 files changed, 15 insertions(+) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 65743d79f..700a95d0c 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -25,6 +25,10 @@ export default function initWorkers( events[type]?.(args); }, }); + + api.closeWorkers = () => { + workers.terminate(); + }; } export function createWorkers(workerPath: string, env: any) { diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index 0b1895e43..20abf8a4a 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -58,6 +58,7 @@ export interface ExecutionContext extends EventEmitter { export interface EngineAPI extends EventEmitter { callWorker: CallWorker; + closeWorkers: () => void; } export interface RuntimeEngine extends EventEmitter { diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index 258af8923..19e5e3f26 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -12,6 +12,10 @@ test.before(() => { initWorkers(api, workerPath); }); +test.after(() => { + api.closeWorkers(); +}); + // TODO should I be tearing down the pool each time? test('initWorkers should add a callWorker function', (t) => { @@ -79,6 +83,8 @@ test('If no env is passed, worker thread should be able to access parent env', a // voila t.is(result, code); + + badAPI.closeWorkers(); }); test('By default, worker thread cannot access parent env if env not set', async (t) => { @@ -94,6 +100,8 @@ test('By default, worker thread cannot access parent env if env not set', async // No fish t.is(result, undefined); + + defaultAPI.closeWorkers(); }); test('Worker thread cannot access parent env if custom env is passted', async (t) => { @@ -112,4 +120,6 @@ test('Worker thread cannot access parent env if custom env is passted', async (t const result2 = await customAPI.callWorker('readEnv', ['NODE_ENV']); t.is(result2, 'production'); + + customAPI.closeWorkers(); }); From 4fb93886c95e5541872a23ad656e3d545dbde0ea Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 16:25:45 +0100 Subject: [PATCH 190/232] engine: make engine creation async --- packages/engine-multi/src/api.ts | 4 +- packages/engine-multi/src/engine.ts | 13 +++- packages/engine-multi/test/api.test.ts | 70 ++++++++++--------- packages/engine-multi/test/engine.test.ts | 54 +++++++------- .../engine-multi/test/integration.test.ts | 48 ++++++------- 5 files changed, 101 insertions(+), 88 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 0928a5b86..b7c2ed48e 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -33,7 +33,7 @@ export type RTEOptions = { // Create the engine and handle user-facing stuff, like options parsing // and defaulting -const createAPI = function (options: RTEOptions = {}) { +const createAPI = async function (options: RTEOptions = {}) { let { repoDir } = options; const logger = options.logger || createLogger('RTE', { level: 'debug' }); @@ -69,7 +69,7 @@ const createAPI = function (options: RTEOptions = {}) { // Note that the engine here always uses the standard worker, the real one // To use a mock, create the engine directly - const engine = createEngine(engineOptions); + const engine = await createEngine(engineOptions); // Return the external API return { diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 1a1cf754a..2335d7615 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -20,6 +20,12 @@ import type { EngineAPI, EventHandler, WorkflowState } from './types'; import type { Logger } from '@openfn/logger'; import type { AutoinstallOptions } from './api/autoinstall'; +// TODO let me deal with the fallout first +const validateworker = (_path?: string) => + new Promise((resolve) => { + setTimeout(() => resolve(), 10); + }); + // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? const createWorkflowEvents = ( @@ -72,7 +78,7 @@ export type EngineOptions = { // tbh this is actually the engine, right, this is where stuff happens // the api file is more about the public api I think // TOOD options MUST have a logger -const createEngine = (options: EngineOptions, workerPath?: string) => { +const createEngine = async (options: EngineOptions, workerPath?: string) => { const states: Record = {}; const contexts: Record = {}; const deferredListeners: Record[]> = {}; @@ -97,6 +103,9 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { workerPath || '../dist/worker/worker.js' ); } + + await validateworker(workerPath); + options.logger!.debug('Loading workers from ', resolvedWorkerPath); const engine = new Engine() as EngineAPI; @@ -124,7 +133,7 @@ const createEngine = (options: EngineOptions, workerPath?: string) => { // TODO maybe engine options is too broad? const executeWrapper = ( plan: ExecutionPlan, - opts: Partial + opts: Partial = {} ) => { options.logger!.debug('executing plan ', plan?.id ?? ''); const workflowId = plan.id!; diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index fa326379d..e9d2de120 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -11,20 +11,19 @@ test.afterEach(() => { logger._reset(); }); -test('create a default engine api without throwing', (t) => { - createAPI(); +test.serial('create a default engine api without throwing', async (t) => { + await createAPI(); t.pass(); }); -test('create an engine api with options without throwing', (t) => { - createAPI({ logger }); - +test.serial('create an engine api with options without throwing', async (t) => { + await createAPI({ logger }); // just a token test to see if the logger is accepted and used t.assert(logger._history.length > 0); }); -test('create an engine api with a limited surface', (t) => { - const api = createAPI({ logger }); +test.serial('create an engine api with a limited surface', async (t) => { + const api = await createAPI({ logger }); const keys = Object.keys(api); // TODO the api will actually probably get a bit bigger than this @@ -35,37 +34,40 @@ test('create an engine api with a limited surface', (t) => { // I won't want to do deep testing on execute here - I just want to make sure the basic // exeuction functionality is working. It's more a test of the api surface than the inner // workings of the job -test('execute should return an event listener and receive workflow-complete', (t) => { - return new Promise((done) => { - const api = createAPI({ - logger, - // Disable compilation - compile: { - skip: true, - }, - }); - - const plan = { - id: 'a', - jobs: [ - { - expression: 'export default [s => s]', - // with no adaptor it shouldn't try to autoinstall +test.serial( + 'execute should return an event listener and receive workflow-complete', + async (t) => { + return new Promise(async (done) => { + const api = await createAPI({ + logger, + // Disable compilation + compile: { + skip: true, }, - ], - }; + }); - const listener = api.execute(plan); - listener.on('workflow-complete', () => { - t.pass('workflow completed'); - done(); + const plan = { + id: 'a', + jobs: [ + { + expression: 'export default [s => s]', + // with no adaptor it shouldn't try to autoinstall + }, + ], + }; + + const listener = api.execute(plan); + listener.on('workflow-complete', () => { + t.pass('workflow completed'); + done(); + }); }); - }); -}); + } +); -test('should listen to workflow-complete', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('should listen to workflow-complete', async (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, // Disable compilation compile: { diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 519670c6c..df00179d2 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -28,8 +28,8 @@ test.afterEach(() => { logger._reset(); }); -test('create an engine', (t) => { - const engine = createEngine(options); +test('create an engine', async (t) => { + const engine = await createEngine(options); t.truthy(engine); t.is(engine.constructor.name, 'Engine'); t.truthy(engine.execute); @@ -38,9 +38,11 @@ test('create an engine', (t) => { t.truthy(engine.emit); }); -test('register a workflow', (t) => { +test.todo('throw if the worker is invalid'); + +test('register a workflow', async (t) => { const plan = { id: 'z' }; - const engine = createEngine(options); + const engine = await createEngine(options); const state = engine.registerWorkflow(plan); @@ -49,9 +51,9 @@ test('register a workflow', (t) => { t.deepEqual(state.plan, plan); }); -test('get workflow state', (t) => { +test('get workflow state', async (t) => { const plan = { id: 'z' } as ExecutionPlan; - const engine = createEngine(options); + const engine = await createEngine(options); const s = engine.registerWorkflow(plan); @@ -60,22 +62,22 @@ test('get workflow state', (t) => { t.deepEqual(state, s); }); -test('use the default worker path', (t) => { - const engine = createEngine({ logger, repoDir: '.' }); +test('use the default worker path', async (t) => { + const engine = await createEngine({ logger, repoDir: '.' }); t.true(engine.workerPath.endsWith('worker/worker.js')); }); // Note that even though this is a nonsense path, we get no error at this point -test('use a custom worker path', (t) => { +test('use a custom worker path', async (t) => { const p = 'jam'; - const engine = createEngine(options, p); + const engine = await createEngine(options, p); t.is(engine.workerPath, p); }); -test('execute with test worker and trigger workflow-complete', (t) => { - return new Promise((done) => { +test('execute with test worker and trigger workflow-complete', async (t) => { + return new Promise(async (done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine(options, p); + const engine = await createEngine(options, p); const plan = { id: 'a', @@ -94,10 +96,10 @@ test('execute with test worker and trigger workflow-complete', (t) => { }); }); -test('execute does not return internal state stuff', (t) => { - return new Promise((done) => { +test('execute does not return internal state stuff', async (t) => { + return new Promise(async (done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine(options, p); + const engine = await createEngine(options, p); const plan = { id: 'a', @@ -108,7 +110,7 @@ test('execute does not return internal state stuff', (t) => { ], }; - const result = engine.execute(plan); + const result = engine.execute(plan, {}); // Execute returns an event listener t.truthy(result.on); t.truthy(result.once); @@ -129,10 +131,10 @@ test('execute does not return internal state stuff', (t) => { }); }); -test('listen to workflow-complete', (t) => { - return new Promise((done) => { +test('listen to workflow-complete', async (t) => { + return new Promise(async (done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine(options, p); + const engine = await createEngine(options, p); const plan = { id: 'a', @@ -154,10 +156,10 @@ test('listen to workflow-complete', (t) => { }); }); -test('call listen before execute', (t) => { - return new Promise((done) => { +test('call listen before execute', async (t) => { + return new Promise(async (done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine(options, p); + const engine = await createEngine(options, p); const plan = { id: 'a', @@ -178,10 +180,10 @@ test('call listen before execute', (t) => { }); }); -test.only('catch and emit errors', (t) => { - return new Promise((done) => { +test('catch and emit errors', async (t) => { + return new Promise(async (done) => { const p = path.resolve('src/test/worker-functions.js'); - const engine = createEngine(options, p); + const engine = await createEngine(options, p); const plan = { id: 'a', diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 5d78199d3..a96bdd6c1 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -30,9 +30,9 @@ const createPlan = (jobs?: any[]) => ({ ], }); -test('trigger workflow-start', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('trigger workflow-start', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, compile: { skip: true, @@ -50,9 +50,9 @@ test('trigger workflow-start', (t) => { }); }); -test('trigger job-start', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('trigger job-start', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, compile: { skip: true, @@ -68,9 +68,9 @@ test('trigger job-start', (t) => { }); }); -test('trigger job-complete', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('trigger job-complete', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, compile: { skip: true, @@ -88,9 +88,9 @@ test('trigger job-complete', (t) => { test.todo('trigger multiple job-completes'); -test('trigger workflow-complete', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('trigger workflow-complete', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, compile: { skip: true, @@ -110,9 +110,9 @@ test('trigger workflow-complete', (t) => { }); }); -test('trigger workflow-log for job logs', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('trigger workflow-log for job logs', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, compile: { skip: true, @@ -135,9 +135,9 @@ test('trigger workflow-log for job logs', (t) => { }); }); -test('compile and run', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('compile and run', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, }); @@ -154,9 +154,9 @@ test('compile and run', (t) => { }); }); -test('evaluate conditional edges', (t) => { - return new Promise((done) => { - const api = createAPI({ +test.serial('evaluate conditional edges', (t) => { + return new Promise(async (done) => { + const api = await createAPI({ logger, }); @@ -187,8 +187,8 @@ test('evaluate conditional edges', (t) => { }); }); -test('preload credentials', (t) => { - return new Promise((done) => { +test.serial('preload credentials', (t) => { + return new Promise(async (done) => { let didCallLoader = true; const loader = (id: string) => @@ -200,7 +200,7 @@ test('preload credentials', (t) => { }, 100); }); - const api = createAPI({ + const api = await createAPI({ logger, resolvers: { credential: loader, From 8e0826f68b0a6db54317a7c1f3b85804977fadae Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 16:32:53 +0100 Subject: [PATCH 191/232] ws-worker: handle async engine creation --- packages/ws-worker/src/mock/runtime-engine.ts | 2 +- packages/ws-worker/src/start.ts | 31 ++++++++++++------- packages/ws-worker/test/api/execute.test.ts | 8 ++--- packages/ws-worker/test/integration.test.ts | 5 +-- .../test/mock/runtime-engine.test.ts | 30 +++++++++--------- 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index 501cc8fe6..e39e325c4 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -40,7 +40,7 @@ export type WorkflowErrorEvent = { message: string; }; -function createMock() { +async function createMock() { const activeWorkflows = {} as Record; const bus = new EventEmitter(); const listeners: Record = {}; diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index f74b3158c..4e6ed2534 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -72,19 +72,26 @@ if (args.lightning === 'mock') { args.secret = WORKER_SECRET; } + +function engineReady(engine) { + createWorker(engine, { + port: args.port, + lightning: args.lightning, + logger, + secret: args.secret, + noLoop: !args.loop, + }); +} + let engine; if (args.mock) { - engine = createMockRTE(); - logger.debug('Mock engine created'); + createMockRTE().then((engine) => { + logger.debug('Mock engine created'); + engineReady(engine); + }); } else { - engine = createRTE({ repoDir: args.repoDir }); - logger.debug('engine created'); + createRTE({ repoDir: args.repoDir }).then((engine) => { + logger.debug('engine created'); + engineReady(engine); + }); } - -createWorker(engine, { - port: args.port, - lightning: args.lightning, - logger, - secret: args.secret, - noLoop: !args.loop, -}); diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 8ea57fe16..08cca0092 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -319,7 +319,7 @@ test('loadCredential should fetch a credential', async (t) => { test('execute should return the final result', async (t) => { const channel = mockChannel(mockEventHandlers); - const engine = createMockRTE(); + const engine = await createMockRTE(); const logger = createMockLogger(); const plan = { @@ -349,7 +349,7 @@ test.only('execute should lazy-load a credential', async (t) => { return {}; }, }); - const engine = createMockRTE(); + const engine = await createMockRTE(); const plan = { id: 'a', @@ -378,7 +378,7 @@ test('execute should lazy-load initial state', async (t) => { return toArrayBuffer({}); }, }); - const engine = createMockRTE(); + const engine = await createMockRTE(); const plan: Partial = { id: 'a', @@ -398,7 +398,7 @@ test('execute should lazy-load initial state', async (t) => { test('execute should call all events on the socket', async (t) => { const logger = createMockLogger(); - const engine = createMockRTE(); + const engine = await createMockRTE(); const events = {}; diff --git a/packages/ws-worker/test/integration.test.ts b/packages/ws-worker/test/integration.test.ts index 48ae516ee..05e38cbd5 100644 --- a/packages/ws-worker/test/integration.test.ts +++ b/packages/ws-worker/test/integration.test.ts @@ -16,10 +16,11 @@ const urls = { lng: 'ws://localhost:7654/worker', }; -test.before(() => { +test.before(async () => { + const engine = await createMockRTE(); // TODO give lightning the same secret and do some validation lng = createLightningServer({ port: 7654 }); - worker = createWorkerServer(createMockRTE('engine'), { + worker = createWorkerServer(engine, { port: 4567, lightning: urls.lng, secret: 'abc', diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index 68413113e..c0a8db0fc 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -19,15 +19,15 @@ const sampleWorkflow = { ], } as ExecutionPlan; -test('getStatus() should should have no active workflows', (t) => { - const engine = create(); +test('getStatus() should should have no active workflows', async (t) => { + const engine = await create(); const { active } = engine.getStatus(); t.is(active, 0); }); test('Dispatch start events for a new workflow', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'workflow-start'); @@ -36,7 +36,7 @@ test('Dispatch start events for a new workflow', async (t) => { }); test('getStatus should report one active workflow', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const { active } = engine.getStatus(); @@ -45,7 +45,7 @@ test('getStatus should report one active workflow', async (t) => { }); test('Dispatch complete events when a workflow completes', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const evt = await waitForEvent( @@ -57,7 +57,7 @@ test('Dispatch complete events when a workflow completes', async (t) => { }); test('Dispatch start events for a job', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'job-start'); @@ -67,7 +67,7 @@ test('Dispatch start events for a job', async (t) => { }); test('Dispatch complete events for a job', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'job-complete'); @@ -78,7 +78,7 @@ test('Dispatch complete events for a job', async (t) => { }); test('mock should evaluate expressions as JSON', async (t) => { - const engine = create(); + const engine = await create(); engine.execute(sampleWorkflow); const evt = await waitForEvent(engine, 'job-complete'); @@ -86,7 +86,7 @@ test('mock should evaluate expressions as JSON', async (t) => { }); test('mock should return initial state as result state', async (t) => { - const engine = create(); + const engine = await create(); const wf = { initialState: { y: 22 }, @@ -103,7 +103,7 @@ test('mock should return initial state as result state', async (t) => { }); test('mock prefers JSON state to initial state', async (t) => { - const engine = create(); + const engine = await create(); const wf = { initialState: { y: 22 }, @@ -121,7 +121,7 @@ test('mock prefers JSON state to initial state', async (t) => { }); test('mock should dispatch log events when evaluating JSON', async (t) => { - const engine = create(); + const engine = await create(); const logs = []; engine.on('log', (l) => { @@ -145,7 +145,7 @@ test('resolve credential before job-start if credential is a string', async (t) return {}; }; - const engine = create(); + const engine = await create(); // @ts-ignore engine.execute(wf, { resolvers: { credential } }); @@ -154,7 +154,7 @@ test('resolve credential before job-start if credential is a string', async (t) }); test('listen to events', async (t) => { - const engine = create(); + const engine = await create(); const called = { 'job-start': false, @@ -197,7 +197,7 @@ test('listen to events', async (t) => { }); test('only listen to events for the correct workflow', async (t) => { - const engine = create(); + const engine = await create(); engine.listen('bobby mcgee', { 'workflow-start': ({ workflowId }) => { @@ -220,7 +220,7 @@ test('do nothing for a job if no expression and adaptor (trigger node)', async ( ], } as ExecutionPlan; - const engine = create(); + const engine = await create(); let didCallEvent = false; From fe17e9aeaaa3e5d9a32229b49a99320315d840ac Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 16:35:04 +0100 Subject: [PATCH 192/232] worker-tests: adapt to async engine --- .../worker/test/integration.test.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 54ec4759f..98973e948 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -25,8 +25,8 @@ const initLightning = () => { lightning = createLightningServer({ port: 9999 }); }; -const initWorker = () => { - engine = createEngine({ +const initWorker = async () => { + engine = await createEngine({ // logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), repoDir: path.resolve('./tmp/repo'), @@ -66,7 +66,7 @@ test('should join attempts queue channel', (t) => { }); test('should run a simple job with no compilation or adaptor', (t) => { - return new Promise((done) => { + return new Promise(async (done) => { initLightning(); lightning.on('attempt:complete', (evt) => { // This will fetch the final dataclip from the attempt @@ -76,7 +76,7 @@ test('should run a simple job with no compilation or adaptor', (t) => { t.pass('completed attempt'); done(); }); - initWorker(); + await initWorker(); lightning.enqueueAttempt({ id: 'a1', @@ -93,7 +93,7 @@ test('should run a simple job with no compilation or adaptor', (t) => { // todo ensure repo is clean // check how we manage the env in cli tests test('run a job with autoinstall of common', (t) => { - return new Promise((done) => { + return new Promise(async (done) => { initLightning(); let autoinstallEvent; @@ -117,7 +117,7 @@ test('run a job with autoinstall of common', (t) => { } }); - initWorker(); + await initWorker(); // listen to events for this attempt engine.listen('a33', { @@ -141,7 +141,7 @@ test('run a job with autoinstall of common', (t) => { // this depends on prior test! test('run a job which does NOT autoinstall common', (t) => { - return new Promise((done, _fail) => { + return new Promise(async (done, _fail) => { initLightning(); lightning.on('attempt:complete', (evt) => { @@ -157,7 +157,7 @@ test('run a job which does NOT autoinstall common', (t) => { } }); - initWorker(); + await initWorker(); // listen to events for this attempt engine.listen('a10', { @@ -182,7 +182,7 @@ test('run a job which does NOT autoinstall common', (t) => { }); test('run a job with initial state', (t) => { - return new Promise((done) => { + return new Promise(async (done) => { const attempt = { id: crypto.randomUUID(), dataclip_id: 's1', @@ -209,7 +209,7 @@ test('run a job with initial state', (t) => { done(); }); - initWorker(); + await initWorker(); // TODO: is there any way I can test the worker behaviour here? // I think I can listen to load-state right? @@ -240,7 +240,7 @@ test('run a job with credentials', (t) => { return app.listen(PORT); }; - return new Promise((done) => { + return new Promise(async (done) => { const server = createServer(); const config = { username: 'logan', @@ -282,13 +282,13 @@ test('run a job with credentials', (t) => { done(); }); - initWorker(); + await initWorker(); lightning.enqueueAttempt(attempt); }); }); test('blacklist a non-openfn adaptor', (t) => { - return new Promise((done) => { + return new Promise(async (done) => { const attempt = { id: crypto.randomUUID(), jobs: [ @@ -309,7 +309,7 @@ test('blacklist a non-openfn adaptor', (t) => { done(); }); - initWorker(); + await initWorker(); lightning.enqueueAttempt(attempt); }); From 6348a34df3a3cf1b03220e726e4d35c4691e200d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 16:48:52 +0100 Subject: [PATCH 193/232] engine: run a little validation script on startup --- .../engine-multi/src/api/validate-worker.ts | 16 ++++++++++++ packages/engine-multi/src/engine.ts | 11 +++----- .../engine-multi/src/test/worker-functions.js | 1 + packages/engine-multi/src/worker/worker.ts | 7 +++-- .../test/api/validate-worker.test.ts | 26 +++++++++++++++++++ packages/engine-multi/test/engine.test.ts | 7 +++-- 6 files changed, 52 insertions(+), 16 deletions(-) create mode 100644 packages/engine-multi/src/api/validate-worker.ts create mode 100644 packages/engine-multi/test/api/validate-worker.test.ts diff --git a/packages/engine-multi/src/api/validate-worker.ts b/packages/engine-multi/src/api/validate-worker.ts new file mode 100644 index 000000000..4f75b9485 --- /dev/null +++ b/packages/engine-multi/src/api/validate-worker.ts @@ -0,0 +1,16 @@ +// TODO let me deal with the fallout first + +import { EngineAPI } from '../types'; + +// Simple vaidation function to ensure that a worker is loaded +// Call a handshake task in a worker thread +// This really jsut validates that the worker path exists + +export default async (api: EngineAPI) => { + try { + await api.callWorker('handshake', []); + } catch { + // Throw a nice error if the worker isn't valid + throw new Error('Invalid worker path'); + } +}; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 2335d7615..cf2e2c09d 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -13,6 +13,7 @@ import { import initWorkers from './api/call-worker'; import createState from './api/create-state'; import execute from './api/execute'; +import validateWorker from './api/validate-worker'; import ExecutionContext from './classes/ExecutionContext'; import type { LazyResolvers } from './api'; @@ -20,12 +21,6 @@ import type { EngineAPI, EventHandler, WorkflowState } from './types'; import type { Logger } from '@openfn/logger'; import type { AutoinstallOptions } from './api/autoinstall'; -// TODO let me deal with the fallout first -const validateworker = (_path?: string) => - new Promise((resolve) => { - setTimeout(() => resolve(), 10); - }); - // For each workflow, create an API object with its own event emitter // this is a bt wierd - what if the emitter went on state instead? const createWorkflowEvents = ( @@ -104,14 +99,14 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { ); } - await validateworker(workerPath); - options.logger!.debug('Loading workers from ', resolvedWorkerPath); const engine = new Engine() as EngineAPI; initWorkers(engine, resolvedWorkerPath); + await validateWorker(engine); + // TODO I think this needs to be like: // take a plan // create, register and return a state object diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index c95a39abb..a2a69440d 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -2,6 +2,7 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; workerpool.worker({ + handshake: () => true, test: (result = 42) => { const { pid, scribble } = process; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index cef1cafb7..5f4a7d987 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -17,10 +17,9 @@ import helper, { createLoggers, publish } from './worker-helper'; import { NotifyEvents } from '@openfn/runtime'; workerpool.worker({ - // TODO: add a startup script to ensure the worker is ok - // if we can't call init, there's something wrong with the worker - // and then we have to abort the engine or something - //init: () => {}, + // startup validation script + handshake: () => true, + run: ( plan: ExecutionPlan, adaptorPaths: Record diff --git a/packages/engine-multi/test/api/validate-worker.test.ts b/packages/engine-multi/test/api/validate-worker.test.ts new file mode 100644 index 000000000..699d75ba6 --- /dev/null +++ b/packages/engine-multi/test/api/validate-worker.test.ts @@ -0,0 +1,26 @@ +import test from 'ava'; +import path from 'node:path'; + +import initWorkers from '../../src/api/call-worker'; +import { EngineAPI } from '../../src/types'; +import validateWorker from '../../src/api/validate-worker'; + +let api = {} as EngineAPI; + +const workerPath = path.resolve('src/test/worker-functions.js'); + +test.beforeEach(() => { + api = {} as EngineAPI; +}); + +test('validate should not throw if the worker path is valid', async (t) => { + initWorkers(api, workerPath); + await t.notThrowsAsync(() => validateWorker(api)); +}); + +test('validate should throw if the worker path is invalid', async (t) => { + initWorkers(api, 'a/b/c.js'); + await t.throwsAsync(() => validateWorker(api), { + message: 'Invalid worker path', + }); +}); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index df00179d2..10da32cc8 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -67,11 +67,10 @@ test('use the default worker path', async (t) => { t.true(engine.workerPath.endsWith('worker/worker.js')); }); -// Note that even though this is a nonsense path, we get no error at this point test('use a custom worker path', async (t) => { - const p = 'jam'; - const engine = await createEngine(options, p); - t.is(engine.workerPath, p); + const workerPath = path.resolve('src/test/worker-functions.js'); + const engine = await createEngine(options, workerPath); + t.is(engine.workerPath, workerPath); }); test('execute with test worker and trigger workflow-complete', async (t) => { From 5f708e3497179886150239aef06c84c72f3faa81 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 19 Oct 2023 17:02:06 +0100 Subject: [PATCH 194/232] worker: types --- packages/ws-worker/src/start.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 4e6ed2534..0813315ab 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -73,7 +73,7 @@ if (args.lightning === 'mock') { args.secret = WORKER_SECRET; } -function engineReady(engine) { +function engineReady(engine: any) { createWorker(engine, { port: args.port, lightning: args.lightning, @@ -83,7 +83,6 @@ function engineReady(engine) { }); } -let engine; if (args.mock) { createMockRTE().then((engine) => { logger.debug('Mock engine created'); From 8937fec902dec6f9b0ccefce5db29b5d7cfb56fc Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 12:19:15 +0100 Subject: [PATCH 195/232] engine: add workerpool tests, update docs --- packages/engine-multi/README.md | 43 +++++++++ packages/engine-multi/src/engine.ts | 2 +- packages/engine-multi/src/test/counter.js | 10 ++ .../engine-multi/src/test/worker-functions.js | 18 ++++ .../test/worker/worker-pool.test.ts | 92 ++++++++++++++++--- 5 files changed, 149 insertions(+), 16 deletions(-) create mode 100644 packages/engine-multi/src/test/counter.js diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md index cd4a25942..6ebd4bd52 100644 --- a/packages/engine-multi/README.md +++ b/packages/engine-multi/README.md @@ -5,3 +5,46 @@ A runtime engine which runs multiple jobs in worker threads. A long-running node service, suitable for integration with a Worker, for executing workflows. Docs to follow + +## Usage + +TODO (describe execute, listen and execute.on) + +## Getting Started + +TODO (overview for devs using this repo) + +## Adaptor Installation + +The engine has an auto-install feature. This will ensure that all required adaptors for a workflow are installed in the local repo before execution begins. + +## Architecture + +The main interface to the engine, API, exposes a very limited and controlled interface to consumers. api.ts provides the main export and is a thin API wrapper around the main implementation. + +The main implementation is in engine.ts, which exposes a much broader interface, with more options. This is potentially dangerous to consumers, but is extremely useful for unit testing here. + +When execute is called and passed a plan, the engine first generates an execution context. This contains an event emitter just for that workflower and some contextualised state. + +Initial state and credentials are at the moment pre-loaded, with a "fully resolved" state object passed into the runtime. The Runtime has the ability to lazy load but implementing lazy loading across the worker_thread interface has proven tricky. + +## Security Considerations + +The engine uses workerpool to maintain a pool of worker threads. + +As workflows come in to be executed, they are passed to workerpool which will pick an idle worker and execute the workflow within it. + +workerpool has no natural environment hardening, which means workflows running in the same thread will share an environment. Globals set in workflow A will be available to workflow B, and by the same token an adaptor loaded for workflow A will be shared with workflow B. + +We have two mitigations to this: + +- The runtime sandbox itself ensures that each job runs in an isolated context. If a job escapes the sandbox, it will have access to the thread's global scope +- Inside the worker thread, we freeze the global scope. This basically means that jobs are unable to write data to the global scope. + +Inside the worker thread, we ensure that: + +- The parent `process.env` is not visible (by default in workerpool the woker will "inherit" the parent env) +- The parent process is not accessible (check this) +- The parent scope is not visible (this is innate in workerpool design). + +After initialisation, the only way that the parent process and child thread can communicate is a) through the sendMessage() interface (which really means the child can only send messages that the parent is expecting), b) through a shared memory buffer (usage of which is limited and controlled by the parent), and c) returning a value from a function execution. diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index cf2e2c09d..28a2ac2f3 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -193,7 +193,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { // How does this work if deferred? }; - engine.emit('test'); + engine.emit('test'); // TODO remove return Object.assign(engine, { workerPath: resolvedWorkerPath, diff --git a/packages/engine-multi/src/test/counter.js b/packages/engine-multi/src/test/counter.js new file mode 100644 index 000000000..a9c3c1d80 --- /dev/null +++ b/packages/engine-multi/src/test/counter.js @@ -0,0 +1,10 @@ +/** + * This simple module declares a local variable + * And allows caller to increment it + * It allows us to test the internal state of modules + */ +let count = 0; + +export const increment = () => ++count; + +export const getCount = () => count; diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index a2a69440d..ebbd39627 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -1,6 +1,9 @@ +import path from 'node:path'; import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; +import { increment } from './counter.js'; + workerpool.worker({ handshake: () => true, test: (result = 42) => { @@ -21,6 +24,7 @@ workerpool.worker({ } return process.env; }, + threadId: () => threadId, // very very simple intepretation of a run function // Most tests should use the mock-worker instead run: (plan, adaptorPaths) => { @@ -53,4 +57,18 @@ workerpool.worker({ throw err; } }, + + // How robust is this? + // Eg is global.Error frozen? + freeze: () => { + Object.freeze(global); + Object.freeze(this); + }, + // Tests of module state across executions + // Ie, does a module get re-initialised between runs? (No.) + incrementStatic: () => increment(), + incrementDynamic: async () => { + const { increment } = await import(path.resolve('src/test/counter.js')); + return increment(); + }, }); diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index ca6d6c656..c1ceeead0 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -1,18 +1,20 @@ +import path from 'node:path'; import test from 'ava'; import workerpool from 'workerpool'; -//some tests of worker stuff +const workerPath = path.resolve('src/test/worker-functions.js'); + let pool; -test.beforeEach(() => { - pool = workerpool.pool({ maxWorkers: 1 }); -}); +const createPool = () => workerpool.pool({ maxWorkers: 1 }); +const createPoolWithFunctions = () => + workerpool.pool(workerPath, { maxWorkers: 1 }); -test.afterEach(() => { - pool.terminate(); -}); +test.afterEach(() => pool.terminate()); test.serial('run an expression inside a worker', async (t) => { + pool = createPool(); + const fn = () => 42; const result = await pool.exec(fn, []); @@ -20,8 +22,35 @@ test.serial('run an expression inside a worker', async (t) => { t.is(result, 42); }); +test.serial('expressions should have the same threadId', async (t) => { + pool = createPoolWithFunctions(); + + const ids = {}; + + const saveThreadId = (id: string) => { + if (!ids[id]) { + ids[id] = 0; + } + ids[id]++; + }; + + // Run 4 jobs and return the threadId for each + // With only one worker thread they should all be the same + await Promise.all([ + pool.exec('threadId', []).then(saveThreadId), + pool.exec('threadId', []).then(saveThreadId), + pool.exec('threadId', []).then(saveThreadId), + pool.exec('threadId', []).then(saveThreadId), + ]); + + const allUsedIds = Object.keys(ids); + + t.is(allUsedIds.length, 1); + t.is(ids[allUsedIds[0]], 4); +}); + test.serial('worker should not have access to host globals', async (t) => { - const pool = workerpool.pool({ maxWorkers: 1 }); + pool = createPool(); global.x = 22; const fn = () => global.x; @@ -33,6 +62,8 @@ test.serial('worker should not have access to host globals', async (t) => { }); test.serial('worker should not mutate host global scope', async (t) => { + pool = createPool(); + t.is(global.x, undefined); const fn = () => { @@ -44,8 +75,10 @@ test.serial('worker should not mutate host global scope', async (t) => { t.is(global.x, undefined); }); -// fails! This is a problem -test.serial.skip('workers should not affect each other', async (t) => { +// This is potentially a security concern for jobs which escape the runtime sandbox +test.serial('workers share a global scope', async (t) => { + pool = createPool(); + t.is(global.x, undefined); const fn1 = () => { @@ -60,17 +93,18 @@ test.serial.skip('workers should not affect each other', async (t) => { const fn2 = () => global.x; - // Call into the same worker and check the scope is still there + // Call into the same worker and reads the global scope again const result = await pool.exec(fn2, []); - // Fails - result is 9 - t.is(result, undefined); + // And yes, the internal global x has a value of 9 + t.is(result, 9); }); -// maybe flaky? -test.serial.skip( +test.serial( 'workers should not affect each other if global scope is frozen', async (t) => { + pool = createPool(); + t.is(global.x, undefined); const fn1 = () => { @@ -92,3 +126,31 @@ test.serial.skip( t.is(result, undefined); } ); + +// test imports inside the worker +// this is basically testing that imported modules do not get re-intialised +test.serial('static imports should share state across runs', async (t) => { + pool = createPoolWithFunctions(); + + const count1 = await pool.exec('incrementStatic', []); + t.is(count1, 1); + + const count2 = await pool.exec('incrementStatic', []); + t.is(count2, 2); + + const count3 = await pool.exec('incrementStatic', []); + t.is(count3, 3); +}); + +test.serial('dynamic imports should share state across runs', async (t) => { + pool = createPoolWithFunctions(); + + const count1 = await pool.exec('incrementDynamic', []); + t.is(count1, 1); + + const count2 = await pool.exec('incrementDynamic', []); + t.is(count2, 2); + + const count3 = await pool.exec('incrementDynamic', []); + t.is(count3, 3); +}); From 266128ee580c9af25f4c2adeb5e78e4461d88704 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 12:47:01 +0100 Subject: [PATCH 196/232] engine: more security tests --- .../engine-multi/src/test/worker-functions.js | 25 ++++- .../test/worker/worker-pool.test.ts | 99 ++++++++++++++----- 2 files changed, 97 insertions(+), 27 deletions(-) diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index ebbd39627..a92a932e7 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -24,10 +24,11 @@ workerpool.worker({ } return process.env; }, + readParentEnv: () => process.parent.env, threadId: () => threadId, // very very simple intepretation of a run function // Most tests should use the mock-worker instead - run: (plan, adaptorPaths) => { + run: (plan, _adaptorPaths) => { const workflowId = plan.id; workerpool.workerEmit({ type: 'worker:workflow-start', @@ -58,12 +59,30 @@ workerpool.worker({ } }, - // How robust is this? - // Eg is global.Error frozen? + // Experiments with freezing the global scope + // We may do this in the actual worker freeze: () => { + // This is not a deep freeze, so eg global.Error is not frozen + // Also some things like Uint8Array are not freezable, so these remain ways to scribble Object.freeze(global); + Object.freeze(globalThis); + + // Note that this is undefined, so this doesn't matter Object.freeze(this); }, + + setGlobalX: (newValue = 42) => { + global.x = newValue; + }, + + getGlobalX: () => global.x, + + writeToGlobalError: (obj) => { + Object.assign(Error, obj); + }, + + getFromGlobalError: (key) => Error[key], + // Tests of module state across executions // Ie, does a module get re-initialised between runs? (No.) incrementStatic: () => increment(), diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index c1ceeead0..8c05cb769 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -7,8 +7,10 @@ const workerPath = path.resolve('src/test/worker-functions.js'); let pool; const createPool = () => workerpool.pool({ maxWorkers: 1 }); -const createPoolWithFunctions = () => - workerpool.pool(workerPath, { maxWorkers: 1 }); + +// note that a dedicated pool does not allow arbitrary code execution +const createDedicatedPool = (opts = {}) => + workerpool.pool(workerPath, { maxWorkers: 1, ...opts }); test.afterEach(() => pool.terminate()); @@ -23,7 +25,7 @@ test.serial('run an expression inside a worker', async (t) => { }); test.serial('expressions should have the same threadId', async (t) => { - pool = createPoolWithFunctions(); + pool = createDedicatedPool(); const ids = {}; @@ -49,6 +51,37 @@ test.serial('expressions should have the same threadId', async (t) => { t.is(ids[allUsedIds[0]], 4); }); +// This is only true by default and is easily overridable +test.serial('thread has access to parent env', async (t) => { + pool = createDedicatedPool(); + + process.env.TEST = 'foobar'; + + const result = await pool.exec('readEnv', ['TEST']); + + t.is(result, 'foobar'); + + delete process.env.TEST; +}); + +test.serial('parent env can be hidden from thread', async (t) => { + pool = createDedicatedPool({ + workerThreadOpts: { + env: { PRIVATE: 'xyz' }, + }, + }); + + process.env.TEST = 'foobar'; + + const result = await pool.exec('readEnv', ['TEST']); + t.is(result, undefined); + + const result2 = await pool.exec('readEnv', ['PRIVATE']); + t.is(result2, 'xyz'); + + delete process.env.TEST; +}); + test.serial('worker should not have access to host globals', async (t) => { pool = createPool(); global.x = 22; @@ -100,37 +133,55 @@ test.serial('workers share a global scope', async (t) => { t.is(result, 9); }); -test.serial( - 'workers should not affect each other if global scope is frozen', - async (t) => { - pool = createPool(); +test.serial('get/set global x', async (t) => { + pool = createDedicatedPool(); - t.is(global.x, undefined); + await pool.exec('setGlobalX', [11]); + const result = await pool.exec('getGlobalX'); - const fn1 = () => { - Object.freeze(global); - global.x = 9; - }; + t.is(result, 11); +}); + +test.serial('get/set global error', async (t) => { + pool = createDedicatedPool(); - // Set a global inside the worker - await pool.exec(fn1, []); + await pool.exec('writeToGlobalError', [{ y: 222 }]); + const result = await pool.exec('getFromGlobalError', ['y']); + + t.is(result, 222); +}); - // (should not affect us outside) - t.is(global.x, undefined); +test.serial('freeze prevents global scope being mutated', async (t) => { + pool = createDedicatedPool(); - const fn2 = () => global.x; + // Freeze the scope + await pool.exec('freeze', []); - // Call into the same worker and check the scope is still there - const result = await pool.exec(fn2, []); + t.is(global.x, undefined); + + await t.throwsAsync(pool.exec('setGlobalX', [11]), { + message: 'Cannot add property x, object is not extensible', + }); +}); - t.is(result, undefined); - } -); +test.serial('freeze does not prevent global Error being mutated', async (t) => { + pool = createDedicatedPool(); + + // Freeze the scope + await pool.exec('freeze', []); + + t.is(global.x, undefined); + + await pool.exec('writeToGlobalError', [{ y: 222 }]); + const result = await pool.exec('getFromGlobalError', ['y']); + + t.is(result, 222); +}); // test imports inside the worker // this is basically testing that imported modules do not get re-intialised test.serial('static imports should share state across runs', async (t) => { - pool = createPoolWithFunctions(); + pool = createDedicatedPool(); const count1 = await pool.exec('incrementStatic', []); t.is(count1, 1); @@ -143,7 +194,7 @@ test.serial('static imports should share state across runs', async (t) => { }); test.serial('dynamic imports should share state across runs', async (t) => { - pool = createPoolWithFunctions(); + pool = createDedicatedPool(); const count1 = await pool.exec('incrementDynamic', []); t.is(count1, 1); From 269ec33a1f46420f224e5736a5a709c176ae5470 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 12:48:03 +0100 Subject: [PATCH 197/232] engine: update worker functions --- packages/engine-multi/src/test/worker-functions.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index a92a932e7..283a8c80b 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -24,7 +24,6 @@ workerpool.worker({ } return process.env; }, - readParentEnv: () => process.parent.env, threadId: () => threadId, // very very simple intepretation of a run function // Most tests should use the mock-worker instead From 0e2d51063915eb0f119399bbc699d19a2c96173b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 13:09:07 +0100 Subject: [PATCH 198/232] engine: allow min-max workers to be passed --- packages/engine-multi/src/api.ts | 6 +++ packages/engine-multi/src/api/call-worker.ts | 25 ++++++++++--- packages/engine-multi/src/engine.ts | 8 +++- .../engine-multi/test/api/call-worker.test.ts | 37 ++++++++++++++----- 4 files changed, 60 insertions(+), 16 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index b7c2ed48e..d30d5eb2c 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -23,6 +23,9 @@ export type RTEOptions = { logger?: Logger; repoDir?: string; + minWorkers?: number; + maxWorkers?: number; + noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? compile?: { skip: true; @@ -65,6 +68,9 @@ const createAPI = async function (options: RTEOptions = {}) { noCompile: options.compile?.skip ?? false, // TODO should we disable autoinstall overrides? autoinstall: options.autoinstall, + + minWorkers: options.minWorkers, + maxWorkers: options.maxWorkers, }; // Note that the engine here always uses the standard worker, the real one diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 700a95d0c..6ba6588b8 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -9,15 +9,21 @@ type WorkerEvent = { [key: string]: any; }; +type WorkerOptions = { + minWorkers?: number; + maxWorkers?: number; + env?: any; +}; + // Adds a `callWorker` function to the API object, which will execute a task in a worker export default function initWorkers( api: EngineAPI, workerPath: string, - env = {} + options: WorkerOptions = {} ) { // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path - const workers = createWorkers(workerPath, env); + const workers = createWorkers(workerPath, options); api.callWorker = (task: string, args: any[] = [], events: any = {}) => workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { @@ -26,13 +32,18 @@ export default function initWorkers( }, }); - api.closeWorkers = () => { - workers.terminate(); - }; + api.closeWorkers = () => workers.terminate(); } -export function createWorkers(workerPath: string, env: any) { +export function createWorkers(workerPath: string, options: WorkerOptions) { + const { + env = {}, + minWorkers = 0, + maxWorkers = 5, // what's a good default here? Keeping it low to be conservative + } = options; + let resolvedWorkerPath; + if (workerPath) { // If a path to the worker has been passed in, just use it verbatim // We use this to pass a mock worker for testing purposes @@ -44,6 +55,8 @@ export function createWorkers(workerPath: string, env: any) { } return workerpool.pool(resolvedWorkerPath, { + minWorkers, + maxWorkers, workerThreadOpts: { // Note that we have to pass this explicitly to run in ava's test runner execArgv: ['--no-warnings', '--experimental-vm-modules'], diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 28a2ac2f3..72800e759 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -67,6 +67,9 @@ export type EngineOptions = { noCompile?: boolean; compile?: {}; // TODO autoinstall?: AutoinstallOptions; + + minWorkers?: number; + maxWorkers?: number; }; // This creates the internal API @@ -103,7 +106,10 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { const engine = new Engine() as EngineAPI; - initWorkers(engine, resolvedWorkerPath); + initWorkers(engine, resolvedWorkerPath, { + minWorkers: options.minWorkers, + maxWorkers: options.maxWorkers, + }); await validateWorker(engine); diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index 19e5e3f26..b864e6da5 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -12,9 +12,7 @@ test.before(() => { initWorkers(api, workerPath); }); -test.after(() => { - api.closeWorkers(); -}); +test.after(() => api.closeWorkers()); // TODO should I be tearing down the pool each time? @@ -70,9 +68,10 @@ test('callWorker should execute in a different process', async (t) => { }); }); -test('If no env is passed, worker thread should be able to access parent env', async (t) => { +test('If null env is passed, worker thread should be able to access parent env', async (t) => { const badAPI = {} as EngineAPI; - initWorkers(badAPI, workerPath, null!); + const env = null; + initWorkers(badAPI, workerPath, { env }); // Set up a special key on process.env const code = '76ytghjs'; @@ -81,15 +80,34 @@ test('If no env is passed, worker thread should be able to access parent env', a // try and read that key inside the thread const result = await badAPI.callWorker('readEnv', ['TEST']); - // voila + // voila, the kingdom is yours t.is(result, code); badAPI.closeWorkers(); }); -test('By default, worker thread cannot access parent env if env not set', async (t) => { +test('By default, worker thread cannot access parent env if env not set (no options arg)', async (t) => { + const defaultAPI = {} as EngineAPI; + + initWorkers(defaultAPI, workerPath /* no options passed*/); + + // Set up a special key on process.env + const code = '76ytghjs'; + process.env.TEST = code; + + // try and read that key inside the thread + const result = await defaultAPI.callWorker('readEnv', ['TEST']); + + // No fish + t.is(result, undefined); + + defaultAPI.closeWorkers(); +}); + +test('By default, worker thread cannot access parent env if env not set (with options arg)', async (t) => { const defaultAPI = {} as EngineAPI; - initWorkers(defaultAPI, workerPath, undefined); + + initWorkers(defaultAPI, workerPath, { maxWorkers: 1 }); // Set up a special key on process.env const code = '76ytghjs'; @@ -106,7 +124,8 @@ test('By default, worker thread cannot access parent env if env not set', async test('Worker thread cannot access parent env if custom env is passted', async (t) => { const customAPI = {} as EngineAPI; - initWorkers(customAPI, workerPath, { NODE_ENV: 'production' }); + const env = { NODE_ENV: 'production' }; + initWorkers(customAPI, workerPath, { env }); // Set up a special key on process.env const code = '76ytghjs'; From 2f07d905e59d2cc75b970734a18a570468457f32 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 14:32:06 +0100 Subject: [PATCH 199/232] engine: pass whitelist down from the API and expand run options to include it --- packages/engine-multi/src/api.ts | 34 +++++++------------ packages/engine-multi/src/api/execute.ts | 28 +++++++-------- packages/engine-multi/src/engine.ts | 11 ++++-- packages/engine-multi/src/types.ts | 3 +- .../engine-multi/src/worker/mock-worker.ts | 2 +- packages/engine-multi/src/worker/worker.ts | 20 +++++++---- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index d30d5eb2c..1f8a1eca2 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -1,10 +1,9 @@ // Creates the public/external API to the runtime // Basically a thin wrapper, with validation, around the engine -import createLogger, { Logger } from '@openfn/logger'; +import createLogger from '@openfn/logger'; -import createEngine from './engine'; -import type { AutoinstallOptions } from './api/autoinstall'; +import createEngine, { EngineOptions } from './engine'; export type State = any; // TODO I want a nice state def with generics @@ -18,21 +17,14 @@ export type LazyResolvers = { expressions?: Resolver; }; -export type RTEOptions = { - resolvers?: LazyResolvers; - logger?: Logger; - repoDir?: string; - - minWorkers?: number; - maxWorkers?: number; - - noCompile?: boolean; // Needed for unit tests to support json expressions. Maybe we shouldn't do this? - compile?: { - skip: true; - }; - - autoinstall?: AutoinstallOptions; -}; +export type RTEOptions = Partial< + Omit & { + // Needed here for unit tests to support json expressions. Would rather exclude tbh + compile?: { + skip?: boolean; + }; + } +>; // Create the engine and handle user-facing stuff, like options parsing // and defaulting @@ -63,6 +55,8 @@ const createAPI = async function (options: RTEOptions = {}) { logger, resolvers: options.resolvers, // TODO should probably default these? repoDir, + // Only allow @openfn/ modules to be imported into runs + whitelist: [/^@openfn/], // TODO should map this down into compile. noCompile: options.compile?.skip ?? false, @@ -81,10 +75,6 @@ const createAPI = async function (options: RTEOptions = {}) { return { execute: engine.execute, listen: engine.listen, - - // expose a hook to listen to internal events - // @ts-ignore - // on: (...args) => engine.on(...args), }; }; diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 6539d0385..182877989 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -28,6 +28,11 @@ const execute = async (context: ExecutionContext) => { await preloadCredentials(state.plan as any, options.resolvers?.credential); } + const runOptions = { + adaptorPaths, + whitelist: options.whitelist, + }; + const events = { [workerEvents.WORKFLOW_START]: (evt: workerEvents.WorkflowStartEvent) => { workflowStart(context, evt); @@ -47,22 +52,15 @@ const execute = async (context: ExecutionContext) => { log(context, evt); }, }; - return callWorker('run', [state.plan, adaptorPaths], events).catch( - (e: any) => { - // TODO what information can I usefully provide here? - // DO I know which job I'm on? - // DO I know the thread id? - // Do I know where the error came from? - - error(context, { workflowId: state.plan.id, error: e }); - - // If the worker file can't be found, we get: - // code: MODULE_NOT_FOUND - // message: cannot find module (worker.js) + return callWorker('run', [state.plan, runOptions], events).catch((e: any) => { + // TODO what information can I usefully provide here? + // DO I know which job I'm on? + // DO I know the thread id? + // Do I know where the error came from? - logger.error(e); - } - ); + error(context, { workflowId: state.plan.id, error: e }); + logger.error(e); + }); }; export default execute; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 72800e759..3ae783086 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -64,12 +64,18 @@ export type EngineOptions = { resolvers?: LazyResolvers; - noCompile?: boolean; - compile?: {}; // TODO + noCompile?: boolean; // TODO deprecate in favour of compile + + // compile?: { // TODO no support yet + // skip?: boolean; + // }; + autoinstall?: AutoinstallOptions; minWorkers?: number; maxWorkers?: number; + + whitelist?: RegExp[]; }; // This creates the internal API @@ -202,6 +208,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { engine.emit('test'); // TODO remove return Object.assign(engine, { + options, workerPath: resolvedWorkerPath, logger: options.logger, registerWorkflow, diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index 20abf8a4a..d41055a5d 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -3,7 +3,6 @@ import { Logger } from '@openfn/logger'; import { ExecutionPlan } from '@openfn/runtime'; import type { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; -import { RTEOptions } from './api'; import { ExternalEvents, EventMap } from './events'; import { EngineOptions } from './engine'; @@ -45,7 +44,7 @@ export type ExecutionContextConstructor = { export interface ExecutionContext extends EventEmitter { constructor(args: ExecutionContextConstructor): ExecutionContext; - options: RTEOptions; // TODO maybe. bring them in here? + options: EngineOptions; state: WorkflowState; logger: Logger; callWorker: CallWorker; diff --git a/packages/engine-multi/src/worker/mock-worker.ts b/packages/engine-multi/src/worker/mock-worker.ts index 551980386..f83d9396a 100644 --- a/packages/engine-multi/src/worker/mock-worker.ts +++ b/packages/engine-multi/src/worker/mock-worker.ts @@ -71,6 +71,6 @@ function mock(plan: MockExecutionPlan) { } workerpool.worker({ - run: async (plan: MockExecutionPlan, _repoDir?: string) => + run: async (plan: MockExecutionPlan, _options?: any) => helper(plan.id, () => mock(plan)), }); diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 5f4a7d987..462ebf43e 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -16,27 +16,33 @@ import type { ExecutionPlan } from '@openfn/runtime'; import helper, { createLoggers, publish } from './worker-helper'; import { NotifyEvents } from '@openfn/runtime'; +type RunOptions = { + adaptorPaths: Record; + whitelist?: RegExp[]; + + // TODO sanitize policy (gets fed to loggers) + // TODO timeout +}; + workerpool.worker({ // startup validation script handshake: () => true, - run: ( - plan: ExecutionPlan, - adaptorPaths: Record - ) => { + run: (plan: ExecutionPlan, runOptions: RunOptions) => { + const { adaptorPaths, whitelist } = runOptions; const { logger, jobLogger } = createLoggers(plan.id!); + const options = { logger, jobLogger, linker: { modules: adaptorPaths, - whitelist: [/^@openfn/], + whitelist, }, callbacks: { // TODO: this won't actually work across the worker boundary // For now I am preloading credentials - // resolveCredential: async (id: string) => { - // }, + // resolveCredential: async (id: string) => {}, notify: (name: NotifyEvents, payload: any) => { // convert runtime notify events to internal engine events publish(plan.id!, `worker:${name}`, payload); From 488ebfa48d7c0a88f0cca6722b4616cb1bea6a5c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 14:48:35 +0100 Subject: [PATCH 200/232] engine: do not try to install blacklisted modules --- packages/engine-multi/src/api.ts | 3 +- packages/engine-multi/src/api/autoinstall.ts | 9 +- packages/engine-multi/src/whitelist.ts | 1 + .../engine-multi/test/api/autoinstall.test.ts | 94 +++++++++++++++++-- 4 files changed, 96 insertions(+), 11 deletions(-) create mode 100644 packages/engine-multi/src/whitelist.ts diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 1f8a1eca2..6615726ae 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -3,6 +3,7 @@ import createLogger from '@openfn/logger'; +import whitelist from './whitelist'; import createEngine, { EngineOptions } from './engine'; export type State = any; // TODO I want a nice state def with generics @@ -56,7 +57,7 @@ const createAPI = async function (options: RTEOptions = {}) { resolvers: options.resolvers, // TODO should probably default these? repoDir, // Only allow @openfn/ modules to be imported into runs - whitelist: [/^@openfn/], + whitelist, // TODO should map this down into compile. noCompile: options.compile?.skip ?? false, diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index bb02806d4..d5fd0356f 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -29,7 +29,7 @@ const pending: Record> = {}; const autoinstall = async (context: ExecutionContext): Promise => { const { logger, state, options } = context; const { plan } = state; - const { repoDir } = options; + const { repoDir, whitelist } = options; const autoinstallOptions = options.autoinstall || {}; const installFn = autoinstallOptions?.handleInstall || install; @@ -55,6 +55,13 @@ const autoinstall = async (context: ExecutionContext): Promise => { // TODO set iteration is weirdly difficult? const paths: ModulePaths = {}; for (const a of adaptors) { + // Ensure that this is not blacklisted + // TODO what if it is? For now we'll log and skip it + if (whitelist && !whitelist.find((r) => r.exec(a))) { + logger.warn('WARNING: autoinstall skipping blacklisted module ', a); + continue; + } + // Return a path name to this module for the linker to use later // TODO this is all a bit rushed const alias = getAliasedName(a); diff --git a/packages/engine-multi/src/whitelist.ts b/packages/engine-multi/src/whitelist.ts new file mode 100644 index 000000000..51f5db26f --- /dev/null +++ b/packages/engine-multi/src/whitelist.ts @@ -0,0 +1 @@ +export default [/^@openfn/]; diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index 67933b9f4..17604753e 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -4,6 +4,7 @@ import { createMockLogger } from '@openfn/logger'; import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; import { AUTOINSTALL_COMPLETE } from '../../src/events'; import ExecutionContext from '../../src/classes/ExecutionContext'; +import whitelist from '../../src/whitelist'; type PackageJson = { name: string; @@ -20,19 +21,26 @@ const mockHandleInstall = async (specifier: string): Promise => const logger = createMockLogger(); -const createContext = (autoinstallOpts?, jobs?: any[]) => +const createContext = ( + autoinstallOpts?, + jobs?: any[], + customWhitelist?: RegExp[] +) => new ExecutionContext({ state: { id: 'x', status: 'pending', options: {}, plan: { - jobs: jobs || [{ adaptor: 'x@1.0.0' }], + jobs: jobs || [{ adaptor: '@openfn/language-common@1.0.0' }], }, }, logger, - callWorker: () => {}, + // @ts-ignore + callWorker: async () => {}, options: { + logger, + whitelist: customWhitelist || whitelist, repoDir: 'tmp/repo', autoinstall: autoinstallOpts || { handleInstall: mockHandleInstall, @@ -41,6 +49,10 @@ const createContext = (autoinstallOpts?, jobs?: any[]) => }, }); +test.afterEach(() => { + logger._reset(); +}); + test('mock is installed: should be installed', async (t) => { const isInstalled = mockIsInstalled({ name: 'repo', @@ -141,13 +153,44 @@ test.serial( } ); +test.serial( + 'autoinstall: do not try to install blacklisted modules', + async (t) => { + let callCount = 0; + + const mockInstall = () => + new Promise((resolve) => { + callCount++; + setTimeout(() => resolve(), 20); + }); + + const job = [ + { + adaptor: 'lodash@1.0.0', + }, + ]; + + const options = { + skipRepoValidation: true, + handleInstall: mockInstall, + handleIsInstalled: async () => false, + }; + + const context = createContext(options, job); + + await autoinstall(context); + + t.is(callCount, 0); + } +); + test.serial('autoinstall: return a map to modules', async (t) => { const jobs = [ { - adaptor: 'common@1.0.0', + adaptor: '@openfn/language-common@1.0.0', }, { - adaptor: 'http@1.0.0', + adaptor: '@openfn/language-http@1.0.0', }, ]; @@ -161,8 +204,41 @@ test.serial('autoinstall: return a map to modules', async (t) => { const result = await autoinstall(context); t.deepEqual(result, { - common: { path: 'tmp/repo/node_modules/common_1.0.0' }, - http: { path: 'tmp/repo/node_modules/http_1.0.0' }, + '@openfn/language-common': { + path: 'tmp/repo/node_modules/@openfn/language-common_1.0.0', + }, + '@openfn/language-http': { + path: 'tmp/repo/node_modules/@openfn/language-http_1.0.0', + }, + }); +}); + +test.serial('autoinstall: support custom whitelist', async (t) => { + const whitelist = [/^y/]; + const jobs = [ + { + // will be ignored + adaptor: 'x@1.0.0', + }, + { + // will be installed + adaptor: 'y@1.0.0', + }, + ]; + + const autoinstallOpts = { + skipRepoValidation: true, + handleInstall: async () => {}, + handleIsInstalled: async () => false, + }; + const context = createContext(autoinstallOpts, jobs, whitelist); + + const result = await autoinstall(context); + + t.deepEqual(result, { + y: { + path: 'tmp/repo/node_modules/y_1.0.0', + }, }); }); @@ -170,7 +246,7 @@ test.serial('autoinstall: emit an event on completion', async (t) => { let event; const jobs = [ { - adaptor: 'common@1.0.0', + adaptor: '@openfn/language-common@1.0.0', }, ]; @@ -188,7 +264,7 @@ test.serial('autoinstall: emit an event on completion', async (t) => { await autoinstall(context); t.truthy(event); - t.is(event.module, 'common'); + t.is(event.module, '@openfn/language-common'); t.is(event.version, '1.0.0'); t.assert(event.duration >= 50); }); From 673cc114fa39e11e9ed1fc13259a42f9b5ec05b1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 15:08:06 +0100 Subject: [PATCH 201/232] engine: update repo env var; big docs update --- packages/engine-multi/README.md | 83 ++++++++++++++++++++++++++++---- packages/engine-multi/src/api.ts | 8 ++- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md index 6ebd4bd52..0a4f8362f 100644 --- a/packages/engine-multi/README.md +++ b/packages/engine-multi/README.md @@ -4,30 +4,95 @@ A runtime engine which runs multiple jobs in worker threads. A long-running node service, suitable for integration with a Worker, for executing workflows. -Docs to follow - ## Usage -TODO (describe execute, listen and execute.on) +The Engine runs Workflows or Execution Plans. A plan MUST have an id. + +Note: An Execution Plan is NOT the same as a Lightning attempt, although there is a 1:1 mapping between them. + +Instantiate a new Engine: + +``` +import createEngine from '@openfn/engine-multi'; +import createLogger from '@openfn/logger'; + +const engine = await createEngine({ + repoDir: '/tmp/openfn/engine-repo', // this is where modules are autoinstalled to + logger: createLogger('ENGINE', { level: 'debug' }) // control log output +}) +``` + +The createEngine function is asynchronous. It will validate that it is connected to a valid dedicated worker file before reporting for duty. The packaged Engine should do this automatically, but it does require an await. + +Execute a job: + +``` +engine.execute(plan) +``` + +`execute` returns an event emitter which you can listen to: + +``` +engine.execute(plan).on('workflow-complete', (event) => { + const { state, duration } = event; + console.log(`Workflow finsihed in ${duration}`ms) +}) +``` + +You can also call the `listen` API to listen to events from a particular workflow. Listen needs a workflow id and an object of events with callbacks: + +``` +engine.listen(plan.id, { + 'workflow-complete', (event) => { + const { state, duration } = event; + console.log(`Workflow finsihed in ${duration}`ms) + } +}); +engine.execute(plan) +``` + +For a full list of events, see `src/events/ts` (the top-level API events are listed at the top) -## Getting Started +## Module Loader Whitelist -TODO (overview for devs using this repo) +A whitelist controls what modules a job is allowed to import. At the moment this is hardcoded in the Engine to modules starting with @openfn. + +This means jobs cannot do `import _ from 'lodash'`. ## Adaptor Installation The engine has an auto-install feature. This will ensure that all required adaptors for a workflow are installed in the local repo before execution begins. -## Architecture +Blacklisted modules are not installed. -The main interface to the engine, API, exposes a very limited and controlled interface to consumers. api.ts provides the main export and is a thin API wrapper around the main implementation. +You can pass the local repo dir through the `repoDir` argument in `createEngine`, or by setting the `ENGINE_REPO_DIR` env var. -The main implementation is in engine.ts, which exposes a much broader interface, with more options. This is potentially dangerous to consumers, but is extremely useful for unit testing here. +## Resolving Execution Plans -When execute is called and passed a plan, the engine first generates an execution context. This contains an event emitter just for that workflower and some contextualised state. +An ExecutionPlan supports lazy-loading of state objects and configuration credentials. If either of these values appears as a string, the Engine will try to resolve them to object values. + +The Engine cannot do this itself: you must pass a set of resolve functions. These can do whatever you like (a typical use-case is to call up to Lightning). Pass resolvers to the execute call: + +``` +const resolvers = { + credential: (id: string) => lightning.loadCredential(id), + dataclip: (id: string) => lightning.loadState(id), +}; +engine.execute(plan, { resolvers }); +``` Initial state and credentials are at the moment pre-loaded, with a "fully resolved" state object passed into the runtime. The Runtime has the ability to lazy load but implementing lazy loading across the worker_thread interface has proven tricky. +## Architecture + +The Engine uses a dedicated worker found in src/worker/worker.ts. Most of the actual logic is in worker-helper.ts, and is shared by both the real worker (which calls out to @openfn/runtime), and the mock worker (which simulates and evals a run). The mock worker is mostly used in unit tests. + +The main interface to the engine, API, exposes a very limited and controlled interface to consumers. api.ts provides the main export and is a thin API wrapper around the main implementation, providing defauls and validation. + +The main implementation is in engine.ts, which exposes a much broader interface, with more options. This is potentially dangerous to consumers, but is extremely useful for unit testing here. For example, the dedicated worker path can be set here, as can the whitelist. + +When execute is called and passed a plan, the engine first generates an execution context. This contains an event emitter just for that workflower and some contextualised state. + ## Security Considerations The engine uses workerpool to maintain a pool of worker threads. diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 6615726ae..24b3451bd 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -35,14 +35,12 @@ const createAPI = async function (options: RTEOptions = {}) { const logger = options.logger || createLogger('RTE', { level: 'debug' }); if (!repoDir) { - if (process.env.OPENFN_RTE_REPO_DIR) { - repoDir = process.env.OPENFN_RTE_REPO_DIR; + if (process.env.ENGINE_REPO_DIR) { + repoDir = process.env.ENGINE_REPO_DIR; } else { repoDir = '/tmp/openfn/repo'; logger.warn('Using default repodir'); - logger.warn( - 'Set env var OPENFN_RTE_REPO_DIR to use a different directory' - ); + logger.warn('Set env var ENGINE_REPO_DIR to use a different directory'); } } From f95d58927ccdb1cd116090a96b7223be0896992f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 15:46:32 +0100 Subject: [PATCH 202/232] worker docs --- packages/ws-worker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index ee9dc8d6e..87f7246cb 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -35,7 +35,7 @@ You can start a dev server (which rebuilds on save) by running: pnpm start:watch ``` -This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `OPENFN_RTE_REPO_DIR` or `/tmp/openfn/repo`. +This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`. To connect to a lightning instance, pass the `-l` flag. From 7e4529e6d9010c17242f108bec218336065027af Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 15:49:16 +0100 Subject: [PATCH 203/232] logger: export sanitize policies --- .changeset/cyan-worms-clap.md | 5 +++++ packages/logger/src/index.ts | 1 + 2 files changed, 6 insertions(+) create mode 100644 .changeset/cyan-worms-clap.md diff --git a/.changeset/cyan-worms-clap.md b/.changeset/cyan-worms-clap.md new file mode 100644 index 000000000..e0b011c87 --- /dev/null +++ b/.changeset/cyan-worms-clap.md @@ -0,0 +1,5 @@ +--- +'@openfn/logger': patch +--- + +Export SanitizePolicies type diff --git a/packages/logger/src/index.ts b/packages/logger/src/index.ts index 8f7cde5f0..1995f4a9f 100644 --- a/packages/logger/src/index.ts +++ b/packages/logger/src/index.ts @@ -11,3 +11,4 @@ export default createLogger; export type { Logger, JSONLog, StringLog } from './logger'; export type { LogOptions, LogLevel } from './options'; +export type { SanitizePolicies } from './sanitize'; From 643e35aeed84c74987ca320b7ddcd90836ad06e8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 16:38:20 +0100 Subject: [PATCH 204/232] worker: feed attempt options through to engine.execute Not as easy as it looked... --- packages/ws-worker/src/api/execute.ts | 22 ++++-- packages/ws-worker/src/api/start-attempt.ts | 59 +++++++------- packages/ws-worker/src/mock/runtime-engine.ts | 12 ++- packages/ws-worker/src/server.ts | 13 ++-- packages/ws-worker/src/types.d.ts | 8 +- .../ws-worker/src/util/convert-attempt.ts | 14 +++- packages/ws-worker/test/api/execute.test.ts | 31 +++++++- .../ws-worker/test/api/start-attempt.test.ts | 17 +++- .../test/mock/runtime-engine.test.ts | 16 ++++ .../test/util/convert-attempt.test.ts | 78 ++++++++++++------- 10 files changed, 188 insertions(+), 82 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 0989fde03..74737f9cb 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -14,10 +14,9 @@ import { RUN_START, RUN_START_PAYLOAD, } from '../events'; -import { Channel } from '../types'; +import { AttemptOptions, Channel } from '../types'; import { getWithReply, stringify } from '../util'; -import type { ExecutionPlan } from '@openfn/runtime'; import type { JSONLog, Logger } from '@openfn/logger'; import { WorkflowCompleteEvent, @@ -25,6 +24,7 @@ import { WorkflowStartEvent, } from '../mock/runtime-engine'; import type { RuntimeEngine, Resolvers } from '@openfn/engine-multi'; +import { ExecutionPlan } from '@openfn/runtime'; const enc = new TextDecoder('utf-8'); @@ -32,6 +32,7 @@ export type AttemptState = { activeRun?: string; activeJob?: string; plan: ExecutionPlan; + options: AttemptOptions; dataclips: Record; // final dataclip id @@ -60,10 +61,10 @@ export function execute( channel: Channel, engine: RuntimeEngine, logger: Logger, - // TODO first thing we'll do here is pull the plan - plan: ExecutionPlan + plan: ExecutionPlan, + options: AttemptOptions = {} ) { - return new Promise(async (resolve) => { + return new Promise(async (resolve, reject) => { logger.info('execute...'); const state: AttemptState = { @@ -72,6 +73,7 @@ export function execute( // to the initial state lastDataclipId: plan.initialState as string | undefined, dataclips: {}, + options, }; const context: Context = { channel, state, logger, onComplete: resolve }; @@ -145,7 +147,15 @@ export function execute( logger.success('dataclip loaded'); logger.debug(plan.initialState); } - engine.execute(plan, { resolvers }); + + try { + engine.execute(plan, { resolvers, ...options }); + } catch (e: any) { + // TODO what if there's an error? + onWorkflowError(context, { workflowId: plan.id!, message: e.message }); + // are we sure we want to re-throw? + reject(e); + } }); } diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/api/start-attempt.ts index c6e96ba20..a6b2ea095 100644 --- a/packages/ws-worker/src/api/start-attempt.ts +++ b/packages/ws-worker/src/api/start-attempt.ts @@ -1,6 +1,6 @@ import convertAttempt from '../util/convert-attempt'; import { getWithReply } from '../util'; -import { Attempt, Channel, Socket } from '../types'; +import { Attempt, AttemptOptions, Channel, Socket } from '../types'; import { ExecutionPlan } from '@openfn/runtime'; import { GET_ATTEMPT } from '../events'; @@ -17,33 +17,35 @@ const joinAttemptChannel = ( attemptId: string, logger: Logger ) => { - return new Promise<{ channel: Channel; plan: ExecutionPlan }>( - (resolve, reject) => { - // TMP - lightning seems to be sending two responses to me - // just for now, I'm gonna gate the handling here - let didReceiveOk = false; + return new Promise<{ + channel: Channel; + plan: ExecutionPlan; + options: AttemptOptions; + }>((resolve, reject) => { + // TMP - lightning seems to be sending two responses to me + // just for now, I'm gonna gate the handling here + let didReceiveOk = false; - // TODO use proper logger - const channelName = `attempt:${attemptId}`; - logger.debug('connecting to ', channelName); - const channel = socket.channel(channelName, { token }); - channel - .join() - .receive('ok', async (e: any) => { - if (!didReceiveOk) { - didReceiveOk = true; - logger.success(`connected to ${channelName}`, e); - const plan = await loadAttempt(channel); - logger.debug('converted attempt as execution plan:', plan); - resolve({ channel, plan }); - } - }) - .receive('error', (err: any) => { - logger.error(`error connecting to ${channelName}`, err); - reject(err); - }); - } - ); + // TODO use proper logger + const channelName = `attempt:${attemptId}`; + logger.debug('connecting to ', channelName); + const channel = socket.channel(channelName, { token }); + channel + .join() + .receive('ok', async (e: any) => { + if (!didReceiveOk) { + didReceiveOk = true; + logger.success(`connected to ${channelName}`, e); + const { plan, options } = await loadAttempt(channel); + logger.debug('converted attempt as execution plan:', plan); + resolve({ channel, plan, options }); + } + }) + .receive('error', (err: any) => { + logger.error(`error connecting to ${channelName}`, err); + reject(err); + }); + }); }; export default joinAttemptChannel; @@ -52,6 +54,5 @@ export async function loadAttempt(channel: Channel) { // first we get the attempt body through the socket const attemptBody = await getWithReply(channel, GET_ATTEMPT); // then we generate the execution plan - const plan = convertAttempt(attemptBody as Attempt); - return plan; + return convertAttempt(attemptBody as Attempt); } diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts index e39e325c4..899e1cfe7 100644 --- a/packages/ws-worker/src/mock/runtime-engine.ts +++ b/packages/ws-worker/src/mock/runtime-engine.ts @@ -127,8 +127,16 @@ async function createMock() { // The mock uses lots of timeouts to make testing a bit easier and simulate asynchronicity const execute = ( xplan: ExecutionPlan, - { resolvers }: { resolvers?: Resolvers } = { resolvers: mockResolvers } + options: { resolvers?: Resolvers; throw?: boolean } = { + resolvers: mockResolvers, + } ) => { + // This is just an easy way to test the options gets fed through to execute + // Also lets me test error handling! + if (options.throw) { + throw new Error('test error'); + } + const { id, jobs, initialState } = xplan; const workflowId = id; activeWorkflows[id!] = true; @@ -143,7 +151,7 @@ async function createMock() { let state = initialState || {}; // Trivial job reducer in our mock for (const job of jobs) { - state = await executeJob(id!, job, state, resolvers); + state = await executeJob(id!, job, state, options.resolvers); } setTimeout(() => { delete activeWorkflows[id!]; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index b0d690713..afb3d53bd 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -62,13 +62,12 @@ function createServer(engine: any, options: ServerOptions = {}) { const startAttempt = async ({ id, token }: CLAIM_ATTEMPT) => { // TODO need to verify the token against LIGHTNING_PUBLIC_KEY - const { channel: attemptChannel, plan } = await joinAttemptChannel( - socket, - token, - id, - logger - ); - execute(attemptChannel, engine, logger, plan); + const { + channel: attemptChannel, + plan, + options, + } = await joinAttemptChannel(socket, token, id, logger); + execute(attemptChannel, engine, logger, plan, options); }; if (!options.noLoop) { diff --git a/packages/ws-worker/src/types.d.ts b/packages/ws-worker/src/types.d.ts index d93de5497..7e08d14ee 100644 --- a/packages/ws-worker/src/types.d.ts +++ b/packages/ws-worker/src/types.d.ts @@ -1,3 +1,4 @@ +import { SanitizePolicies } from '@openfn/logger'; import type { Socket as PhxSocket, Channel as PhxChannel } from 'phoenix'; export { Socket }; @@ -47,7 +48,12 @@ export type Attempt = { jobs: Node[]; edges: Edge[]; - options?: Record; // TODO type the expected options + options?: AttemptOptions; +}; + +export type AttemptOptions = { + timeout?: number; + sanitize?: SanitizePolicies; }; export type CancelablePromise = Promise & { diff --git a/packages/ws-worker/src/util/convert-attempt.ts b/packages/ws-worker/src/util/convert-attempt.ts index bb2b5570d..c75a5afc2 100644 --- a/packages/ws-worker/src/util/convert-attempt.ts +++ b/packages/ws-worker/src/util/convert-attempt.ts @@ -1,13 +1,16 @@ import crypto from 'node:crypto'; import type { - ExecutionPlan, JobNode, JobNodeID, JobEdge, + ExecutionPlan, } from '@openfn/runtime'; -import { Attempt } from '../types'; +import { Attempt, AttemptOptions } from '../types'; -export default (attempt: Attempt): ExecutionPlan => { +export default ( + attempt: Attempt +): { plan: ExecutionPlan; options: AttemptOptions } => { + const options = attempt.options || {}; const plan: Partial = { id: attempt.id, }; @@ -92,5 +95,8 @@ export default (attempt: Attempt): ExecutionPlan => { plan.jobs = Object.values(nodes); - return plan as ExecutionPlan; + return { + plan: plan as ExecutionPlan, + options, + }; }; diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 08cca0092..7f22c28a2 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -336,8 +336,37 @@ test('execute should return the final result', async (t) => { t.deepEqual(result, { done: true }); }); +test('execute should feed options to the engine', async (t) => { + const channel = mockChannel(mockEventHandlers); + const engine = await createMockRTE(); + const logger = createMockLogger(); + + const plan = { + id: 'a', + jobs: [ + { + expression: JSON.stringify({ done: true }), + }, + ], + }; + + const options = { + throw: true, + }; + + // TODO what do we actually want to do if engine.execute throws? + // Need to to someting... + + try { + await execute(channel, engine, logger, plan, options); + } catch (e) { + t.is(e.message, 'test error'); + t.pass(); + } +}); + // TODO this is more of an engine test really, but worth having I suppose -test.only('execute should lazy-load a credential', async (t) => { +test('execute should lazy-load a credential', async (t) => { const logger = createMockLogger(); let didCallCredentials = false; diff --git a/packages/ws-worker/test/api/start-attempt.test.ts b/packages/ws-worker/test/api/start-attempt.test.ts index e5d92197c..064067469 100644 --- a/packages/ws-worker/test/api/start-attempt.test.ts +++ b/packages/ws-worker/test/api/start-attempt.test.ts @@ -21,14 +21,20 @@ test('loadAttempt should get the attempt body', async (t) => { t.true(didCallGetAttempt); }); -test('loadAttempt should return an execution plan', async (t) => { - const attempt = attempts['attempt-1']; +test('loadAttempt should return an execution plan and options', async (t) => { + const attempt = { + ...attempts['attempt-1'], + options: { + sanitize: 'obfuscate', + timeout: 10, + }, + }; const channel = mockChannel({ [GET_ATTEMPT]: () => attempt, }); - const plan = await loadAttempt(channel); + const { plan, options } = await loadAttempt(channel); t.like(plan, { id: 'attempt-1', jobs: [ @@ -40,6 +46,7 @@ test('loadAttempt should return an execution plan', async (t) => { }, ], }); + t.deepEqual(options, attempt.options); }); test('should join an attempt channel with a token', async (t) => { @@ -50,11 +57,12 @@ test('should join an attempt channel with a token', async (t) => { join: () => ({ status: 'ok' }), [GET_ATTEMPT]: () => ({ id: 'a', + options: { timeout: 10 }, }), }), }); - const { channel, plan } = await joinAttemptChannel( + const { channel, plan, options } = await joinAttemptChannel( socket, 'x.y.z', 'a', @@ -63,6 +71,7 @@ test('should join an attempt channel with a token', async (t) => { t.truthy(channel); t.deepEqual(plan, { id: 'a', jobs: [] }); + t.deepEqual(options, { timeout: 10 }); }); test('should fail to join an attempt channel with an invalid token', async (t) => { diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts index c0a8db0fc..830aff201 100644 --- a/packages/ws-worker/test/mock/runtime-engine.test.ts +++ b/packages/ws-worker/test/mock/runtime-engine.test.ts @@ -135,6 +135,22 @@ test('mock should dispatch log events when evaluating JSON', async (t) => { t.deepEqual(logs[1].message, ['Parsing expression as JSON state']); }); +test('mock should throw if the magic option is passed', async (t) => { + const engine = await create(); + + const logs = []; + engine.on('log', (l) => { + logs.push(l); + }); + + await t.throwsAsync( + async () => engine.execute(sampleWorkflow, { throw: true }), + { + message: 'test error', + } + ); +}); + test('resolve credential before job-start if credential is a string', async (t) => { const wf = clone(sampleWorkflow); wf.jobs[0].configuration = 'x'; diff --git a/packages/ws-worker/test/util/convert-attempt.test.ts b/packages/ws-worker/test/util/convert-attempt.test.ts index ea02d294b..5f86e3dd8 100644 --- a/packages/ws-worker/test/util/convert-attempt.test.ts +++ b/packages/ws-worker/test/util/convert-attempt.test.ts @@ -43,14 +43,34 @@ test('convert a single job', (t) => { triggers: [], edges: [], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [createJob()], }); }); +test('convert a single job with options', (t) => { + const attempt: Partial = { + id: 'w', + jobs: [createNode()], + triggers: [], + edges: [], + options: { + sanitize: 'obfuscate', + timeout: 10, + }, + }; + const { plan, options } = convertAttempt(attempt as Attempt); + + t.deepEqual(plan, { + id: 'w', + jobs: [createJob()], + }); + t.deepEqual(options, attempt.options); +}); + // Note idk how lightningg will handle state/defaults on a job // but this is what we'll do right now test('convert a single job with data', (t) => { @@ -60,24 +80,26 @@ test('convert a single job with data', (t) => { triggers: [], edges: [], }; - const result = convertAttempt(attempt as Attempt); + const { plan, options } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [createJob({ state: { data: { x: 22 } } })], }); + t.deepEqual(options, {}); }); test('Accept a partial attempt object', (t) => { const attempt: Partial = { id: 'w', }; - const result = convertAttempt(attempt as Attempt); + const { plan, options } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [], }); + t.deepEqual(options, {}); }); test('handle dataclip_id', (t) => { @@ -85,9 +107,9 @@ test('handle dataclip_id', (t) => { id: 'w', dataclip_id: 'xyz', }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', initialState: 'xyz', jobs: [], @@ -99,9 +121,9 @@ test('handle starting_node_id', (t) => { id: 'w', starting_node_id: 'j1', }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', start: 'j1', jobs: [], @@ -115,9 +137,9 @@ test('convert a single trigger', (t) => { jobs: [], edges: [], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ { @@ -135,9 +157,9 @@ test('ignore a single edge', (t) => { triggers: [], edges: [createEdge('a', 'b')], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [], }); @@ -156,9 +178,9 @@ test('convert a single trigger with an edge', (t) => { }, ], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ { @@ -190,9 +212,9 @@ test('convert a single trigger with two edges', (t) => { }, ], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ { @@ -222,9 +244,9 @@ test('convert a disabled trigger', (t) => { }, ], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ { @@ -243,9 +265,9 @@ test('convert two linked jobs', (t) => { triggers: [], edges: [createEdge('a', 'b')], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [createJob({ id: 'a', next: { b: true } }), createJob({ id: 'b' })], }); @@ -263,9 +285,9 @@ test('convert a job with two upstream jobs', (t) => { triggers: [], edges: [createEdge('a', 'x'), createEdge('b', 'x')], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ createJob({ id: 'a', next: { x: true } }), @@ -283,9 +305,9 @@ test('convert two linked jobs with an edge condition', (t) => { triggers: [], edges: [createEdge('a', 'b', { condition })], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ createJob({ id: 'a', next: { b: { condition } } }), @@ -301,9 +323,9 @@ test('convert two linked jobs with a disabled edge', (t) => { triggers: [], edges: [createEdge('a', 'b', { enabled: false })], }; - const result = convertAttempt(attempt as Attempt); + const { plan } = convertAttempt(attempt as Attempt); - t.deepEqual(result, { + t.deepEqual(plan, { id: 'w', jobs: [ createJob({ id: 'a', next: { b: { disabled: true } } }), From d5ad41ba8075094218216cb025432925a1ddcf86 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 20 Oct 2023 16:50:26 +0100 Subject: [PATCH 205/232] engine: support sanitize option (and a bit of option reworking) --- packages/engine-multi/src/api.ts | 5 +- .../src/classes/ExecutionContext.ts | 4 +- packages/engine-multi/src/engine.ts | 15 +- packages/engine-multi/src/types.ts | 17 +- .../engine-multi/src/worker/worker-helper.ts | 9 +- packages/engine-multi/src/worker/worker.ts | 8 +- .../engine-multi/test/integration.test.ts | 7 +- pnpm-lock.yaml | 399 +++++++++++++++++- 8 files changed, 439 insertions(+), 25 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 24b3451bd..8d0bf95a1 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -52,7 +52,10 @@ const createAPI = async function (options: RTEOptions = {}) { const engineOptions = { logger, - resolvers: options.resolvers, // TODO should probably default these? + + // TODO should resolvers be set here on passed to execute? + // They do feel more "global" + // resolvers: options.resolvers, // TODO should probably default these? repoDir, // Only allow @openfn/ modules to be imported into runs whitelist, diff --git a/packages/engine-multi/src/classes/ExecutionContext.ts b/packages/engine-multi/src/classes/ExecutionContext.ts index ec5abe3ae..09892e38a 100644 --- a/packages/engine-multi/src/classes/ExecutionContext.ts +++ b/packages/engine-multi/src/classes/ExecutionContext.ts @@ -4,9 +4,9 @@ import type { WorkflowState, CallWorker, ExecutionContextConstructor, + ExecutionContextOptions, } from '../types'; import type { Logger } from '@openfn/logger'; -import type { EngineOptions } from '../engine'; /** * The ExeuctionContext class wraps an event emitter with some useful context @@ -20,7 +20,7 @@ export default class ExecutionContext extends EventEmitter { state: WorkflowState; logger: Logger; callWorker: CallWorker; - options: EngineOptions; + options: ExecutionContextOptions; constructor({ state, diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 3ae783086..cf89369b8 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -2,7 +2,6 @@ import { EventEmitter } from 'node:events'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import type { ExecutionPlan } from '@openfn/runtime'; - import { JOB_COMPLETE, JOB_START, @@ -16,6 +15,7 @@ import execute from './api/execute'; import validateWorker from './api/validate-worker'; import ExecutionContext from './classes/ExecutionContext'; +import type { SanitizePolicies } from '@openfn/logger'; import type { LazyResolvers } from './api'; import type { EngineAPI, EventHandler, WorkflowState } from './types'; import type { Logger } from '@openfn/logger'; @@ -78,6 +78,11 @@ export type EngineOptions = { whitelist?: RegExp[]; }; +export type ExecuteOptions = { + sanitize?: SanitizePolicies; + resolvers?: LazyResolvers; +}; + // This creates the internal API // tbh this is actually the engine, right, this is where stuff happens // the api file is more about the public api I think @@ -138,10 +143,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { // TODO too much logic in this execute function, needs farming out // I don't mind having a wrapper here but it must be super thin // TODO maybe engine options is too broad? - const executeWrapper = ( - plan: ExecutionPlan, - opts: Partial = {} - ) => { + const executeWrapper = (plan: ExecutionPlan, opts: ExecuteOptions = {}) => { options.logger!.debug('executing plan ', plan?.id ?? ''); const workflowId = plan.id!; // TODO throw if plan is invalid @@ -155,7 +157,8 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { callWorker: engine.callWorker, options: { ...options, - ...opts, + sanitize: opts.sanitize, + resolvers: opts.resolvers, }, }); diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index d41055a5d..55b349157 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -1,11 +1,12 @@ // ok first of allI want to capture the key interfaces -import { Logger } from '@openfn/logger'; -import { ExecutionPlan } from '@openfn/runtime'; -import type { EventEmitter } from 'node:events'; import workerpool from 'workerpool'; -import { ExternalEvents, EventMap } from './events'; -import { EngineOptions } from './engine'; +import type { Logger, SanitizePolicies } from '@openfn/logger'; +import type { ExecutionPlan } from '@openfn/runtime'; +import type { EventEmitter } from 'node:events'; + +import type { ExternalEvents, EventMap } from './events'; +import type { EngineOptions } from './engine'; export type Resolver = (id: string) => Promise; @@ -39,7 +40,11 @@ export type ExecutionContextConstructor = { state: WorkflowState; logger: Logger; callWorker: CallWorker; - options: EngineOptions; + options: ExecutionContextOptions; +}; + +export type ExecutionContextOptions = EngineOptions & { + sanitize?: SanitizePolicies; }; export interface ExecutionContext extends EventEmitter { diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts index d09ab7349..5b3f41625 100644 --- a/packages/engine-multi/src/worker/worker-helper.ts +++ b/packages/engine-multi/src/worker/worker-helper.ts @@ -3,11 +3,14 @@ import workerpool from 'workerpool'; import { threadId } from 'node:worker_threads'; -import createLogger from '@openfn/logger'; +import createLogger, { SanitizePolicies } from '@openfn/logger'; import * as workerEvents from './events'; -export const createLoggers = (workflowId: string) => { +export const createLoggers = ( + workflowId: string, + sanitize?: SanitizePolicies +) => { const log = (message: string) => { // Apparently the json log stringifies the message // We don't really want it to do that @@ -32,11 +35,13 @@ export const createLoggers = (workflowId: string) => { logger: emitter, level: 'debug', json: true, + sanitize, }); const jobLogger = createLogger('JOB', { logger: emitter, level: 'debug', json: true, + sanitize, }); return { logger, jobLogger }; diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 462ebf43e..42da9edc7 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -13,14 +13,14 @@ import workerpool from 'workerpool'; import run from '@openfn/runtime'; import type { ExecutionPlan } from '@openfn/runtime'; +import type { SanitizePolicies } from '@openfn/logger'; import helper, { createLoggers, publish } from './worker-helper'; import { NotifyEvents } from '@openfn/runtime'; type RunOptions = { adaptorPaths: Record; whitelist?: RegExp[]; - - // TODO sanitize policy (gets fed to loggers) + sanitize: SanitizePolicies; // TODO timeout }; @@ -29,8 +29,8 @@ workerpool.worker({ handshake: () => true, run: (plan: ExecutionPlan, runOptions: RunOptions) => { - const { adaptorPaths, whitelist } = runOptions; - const { logger, jobLogger } = createLoggers(plan.id!); + const { adaptorPaths, whitelist, sanitize } = runOptions; + const { logger, jobLogger } = createLoggers(plan.id!, sanitize); const options = { logger, diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index a96bdd6c1..d0630a511 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -202,10 +202,13 @@ test.serial('preload credentials', (t) => { const api = await createAPI({ logger, + }); + + const options = { resolvers: { credential: loader, }, - }); + }; const jobs = [ { @@ -216,7 +219,7 @@ test.serial('preload credentials', (t) => { const plan = createPlan(jobs); - api.execute(plan).on('workflow-complete', ({ state }) => { + api.execute(plan, options).on('workflow-complete', ({ state }) => { t.true(didCallLoader); done(); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 751c7c54b..ddef27cf8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -157,6 +157,9 @@ importers: '@openfn/language-common_latest': specifier: npm:@openfn/language-common@^1.11.1 version: /@openfn/language-common@1.11.1 + '@openfn/language-http_latest': + specifier: npm:@openfn/language-http@^5.0.4 + version: /@openfn/language-http@5.0.4 lodash_latest: specifier: npm:lodash@^4.17.21 version: /lodash@4.17.21 @@ -1669,6 +1672,22 @@ packages: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] + /@openfn/language-http@5.0.4: + resolution: {integrity: sha512-zuMlJyORxBps0KO+93a3kVBRzStGwYVNAOEl7GgvO6Z96YBO7/K2NiqSyMHTyQeIyCLUBGmrEv1AAuk4pXxKAg==} + dependencies: + '@openfn/language-common': 1.11.1 + cheerio: 1.0.0-rc.12 + cheerio-tableparser: 1.0.1 + csv-parse: 4.16.3 + fast-safe-stringify: 2.1.1 + form-data: 3.0.1 + lodash: 4.17.21 + request: 2.88.2 + tough-cookie: 4.1.3 + transitivePeerDependencies: + - debug + dev: false + /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -2121,6 +2140,15 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 + /ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + dev: false + /ajv@8.12.0: resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} dependencies: @@ -2265,6 +2293,17 @@ packages: resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} engines: {node: '>=12'} + /asn1@0.2.6: + resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /assert-plus@1.0.0: + resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} + engines: {node: '>=0.8'} + dev: false + /assign-symbols@1.0.0: resolution: {integrity: sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==} engines: {node: '>=0.10.0'} @@ -2420,6 +2459,14 @@ packages: fast-glob: 3.3.1 dev: true + /aws-sign2@0.7.0: + resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} + dev: false + + /aws4@1.12.0: + resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} + dev: false + /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: @@ -2473,6 +2520,12 @@ packages: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} dev: true + /bcrypt-pbkdf@1.0.2: + resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} + dependencies: + tweetnacl: 0.14.5 + dev: false + /bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: true @@ -2524,6 +2577,10 @@ packages: /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2708,6 +2765,10 @@ packages: engines: {node: '>=6'} dev: true + /caseless@0.12.0: + resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + dev: false + /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -2741,6 +2802,34 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio-tableparser@1.0.1: + resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -2991,6 +3080,10 @@ packages: resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} dev: false + /core-util-is@1.0.2: + resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} + dev: false + /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -3041,6 +3134,21 @@ packages: which: 2.0.2 dev: true + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.1.1 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -3057,7 +3165,6 @@ packages: /csv-parse@4.16.3: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} - dev: true /csv-parse@5.5.2: resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} @@ -3093,6 +3200,13 @@ packages: dependencies: array-find-index: 1.0.2 + /dashdash@1.14.1: + resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} + engines: {node: '>=0.10'} + dependencies: + assert-plus: 1.0.0 + dev: false + /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -3277,6 +3391,33 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3300,6 +3441,13 @@ packages: /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + /ecc-jsbn@0.1.2: + resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} + dependencies: + jsbn: 0.1.1 + safer-buffer: 2.1.2 + dev: false + /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3338,6 +3486,11 @@ packages: ansi-colors: 4.1.3 dev: true + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -4024,6 +4177,10 @@ packages: is-extendable: 1.0.1 dev: true + /extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + dev: false + /extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} dev: true @@ -4052,6 +4209,11 @@ packages: - supports-color dev: true + /extsprintf@1.3.0: + resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} + engines: {'0': node >=0.6.0} + dev: false + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false @@ -4088,6 +4250,10 @@ packages: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} dev: false + /fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + dev: false + /fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} dev: true @@ -4226,6 +4392,19 @@ packages: signal-exit: 4.0.2 dev: true + /forever-agent@0.6.1: + resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} + dev: false + + /form-data@2.3.3: + resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} + engines: {node: '>= 0.12'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data@2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -4235,6 +4414,15 @@ packages: mime-types: 2.1.35 dev: true + /form-data@3.0.1: + resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4358,6 +4546,12 @@ packages: engines: {node: '>=0.10.0'} dev: true + /getpass@0.1.7: + resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} + dependencies: + assert-plus: 1.0.0 + dev: false + /glob-parent@3.1.0: resolution: {integrity: sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==} dependencies: @@ -4464,6 +4658,20 @@ packages: through2: 2.0.5 dev: true + /har-schema@2.0.0: + resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} + engines: {node: '>=4'} + dev: false + + /har-validator@5.1.5: + resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} + engines: {node: '>=6'} + deprecated: this library is no longer supported + dependencies: + ajv: 6.12.6 + har-schema: 2.0.0 + dev: false + /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -4549,6 +4757,15 @@ packages: lru-cache: 7.18.3 dev: true + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4617,6 +4834,15 @@ packages: - supports-color dev: true + /http-signature@1.2.0: + resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} + engines: {node: '>=0.8', npm: '>=1.3.7'} + dependencies: + assert-plus: 1.0.0 + jsprim: 1.4.2 + sshpk: 1.18.0 + dev: false + /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -5010,6 +5236,10 @@ packages: has-symbols: 1.0.3 dev: true + /is-typedarray@1.0.0: + resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} + dev: false + /is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -5063,6 +5293,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /isstream@0.1.2: + resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} + dev: false + /jackspeak@2.2.2: resolution: {integrity: sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==} engines: {node: '>=14'} @@ -5103,6 +5337,10 @@ packages: argparse: 2.0.1 dev: true + /jsbn@0.1.1: + resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} + dev: false + /json-diff@1.0.6: resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} hasBin: true @@ -5116,10 +5354,22 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true + /json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + dev: false + /json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} dev: false + /json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + dev: false + + /json-stringify-safe@5.0.1: + resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} + dev: false + /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5143,6 +5393,16 @@ packages: underscore: 1.12.1 dev: true + /jsprim@1.4.2: + resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} + engines: {node: '>=0.6.0'} + dependencies: + assert-plus: 1.0.0 + extsprintf: 1.3.0 + json-schema: 0.4.0 + verror: 1.10.0 + dev: false + /keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} @@ -5883,6 +6143,16 @@ packages: path-key: 3.1.1 dev: true + /nth-check@2.1.1: + resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} + dependencies: + boolbase: 1.0.0 + dev: false + + /oauth-sign@0.9.0: + resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} + dev: false + /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6152,6 +6422,19 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6234,6 +6517,10 @@ packages: through2: 2.0.5 dev: true + /performance-now@2.1.0: + resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} + dev: false + /phoenix@1.7.7: resolution: {integrity: sha512-moAN6e4Z16x/x1nswUpnTR2v5gm7HsI7eluZ2YnYUUsBNzi3cY/5frmiJfXIEi877IQAafzTfp8hd6vEUMme+w==} dev: false @@ -6468,6 +6755,10 @@ packages: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} dev: true + /psl@1.9.0: + resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} + dev: false + /pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -6498,6 +6789,11 @@ packages: side-channel: 1.0.4 dev: false + /qs@6.5.3: + resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} + engines: {node: '>=0.6'} + dev: false + /query-string@8.1.0: resolution: {integrity: sha512-BFQeWxJOZxZGix7y+SByG3F36dA0AbTy9o6pSmKFcFz7DAj0re9Frkty3saBn3nHo3D0oZJ/+rx3r8H8r8Jbpw==} engines: {node: '>=14.16'} @@ -6507,6 +6803,10 @@ packages: split-on-first: 3.0.0 dev: true + /querystringify@2.2.0: + resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} + dev: false + /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6691,6 +6991,33 @@ packages: engines: {node: '>=0.10'} dev: true + /request@2.88.2: + resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} + engines: {node: '>= 6'} + deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 + dependencies: + aws-sign2: 0.7.0 + aws4: 1.12.0 + caseless: 0.12.0 + combined-stream: 1.0.8 + extend: 3.0.2 + forever-agent: 0.6.1 + form-data: 2.3.3 + har-validator: 5.1.5 + http-signature: 1.2.0 + is-typedarray: 1.0.0 + isstream: 0.1.2 + json-stringify-safe: 5.0.1 + mime-types: 2.1.35 + oauth-sign: 0.9.0 + performance-now: 2.1.0 + qs: 6.5.3 + safe-buffer: 5.2.1 + tough-cookie: 2.5.0 + tunnel-agent: 0.6.0 + uuid: 3.4.0 + dev: false + /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -6704,6 +7031,10 @@ packages: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -7127,6 +7458,22 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + /sshpk@1.18.0: + resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} + engines: {node: '>=0.10.0'} + hasBin: true + dependencies: + asn1: 0.2.6 + assert-plus: 1.0.0 + bcrypt-pbkdf: 1.0.2 + dashdash: 1.14.1 + ecc-jsbn: 0.1.2 + getpass: 0.1.7 + jsbn: 0.1.1 + safer-buffer: 2.1.2 + tweetnacl: 0.14.5 + dev: false + /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7458,6 +7805,24 @@ packages: nopt: 1.0.10 dev: true + /tough-cookie@2.5.0: + resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} + engines: {node: '>=0.8'} + dependencies: + psl: 1.9.0 + punycode: 2.3.0 + dev: false + + /tough-cookie@4.1.3: + resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} + engines: {node: '>=6'} + dependencies: + psl: 1.9.0 + punycode: 2.3.0 + universalify: 0.2.0 + url-parse: 1.5.10 + dev: false + /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false @@ -7717,6 +8082,16 @@ packages: yargs: 17.7.2 dev: true + /tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /tweetnacl@0.14.5: + resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} + dev: false + /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -7831,6 +8206,11 @@ packages: engines: {node: '>= 4.0.0'} dev: true + /universalify@0.2.0: + resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} + engines: {node: '>= 4.0.0'} + dev: false + /unix-crypt-td-js@1.1.4: resolution: {integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==} dev: true @@ -7868,6 +8248,13 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false + /url-parse@1.5.10: + resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} + dependencies: + querystringify: 2.2.0 + requires-port: 1.0.0 + dev: false + /use@3.1.1: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} @@ -7886,7 +8273,6 @@ packages: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. hasBin: true - dev: true /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -7909,6 +8295,15 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + /verror@1.10.0: + resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} + engines: {'0': node >=0.6.0} + dependencies: + assert-plus: 1.0.0 + core-util-is: 1.0.2 + extsprintf: 1.3.0 + dev: false + /wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} dependencies: From c7d3321b15aa68238044ad572bcb472ec4c612b5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 24 Oct 2023 14:20:38 +0200 Subject: [PATCH 206/232] worker-tests: experimental dummy repo --- .../@openfn/stateful-test_1.0.0/index.js | 6 +++ .../@openfn/stateful-test_1.0.0/package.json | 7 ++++ .../worker/dummy-repo/package.json | 8 ++++ .../worker/test/integration.test.ts | 42 ++++++++++++++++++- 4 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/index.js create mode 100644 integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/package.json create mode 100644 integration-tests/worker/dummy-repo/package.json diff --git a/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/index.js b/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/index.js new file mode 100644 index 000000000..1075063aa --- /dev/null +++ b/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/index.js @@ -0,0 +1,6 @@ +import crypto from 'node:crypto'; +export{ threadId } from 'node:worker_threads'; + +export const clientId = crypto.randomUUID(); + +export const fn = (f) => (s) => f(s); \ No newline at end of file diff --git a/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/package.json b/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/package.json new file mode 100644 index 000000000..87680af26 --- /dev/null +++ b/integration-tests/worker/dummy-repo/node_modules/@openfn/stateful-test_1.0.0/package.json @@ -0,0 +1,7 @@ +{ + "name": "@openfn/stateful-test", + "version": "1.0.0", + "type": "module", + "main": "index.js", + "private": true +} diff --git a/integration-tests/worker/dummy-repo/package.json b/integration-tests/worker/dummy-repo/package.json new file mode 100644 index 000000000..782ec5ed7 --- /dev/null +++ b/integration-tests/worker/dummy-repo/package.json @@ -0,0 +1,8 @@ +{ + "name": "test-repo", + "private": true, + "version": "1.0.0", + "dependencies": { + "@openfn/stateful-test_1.0.0": "@npm:@openfn/stateful-test@1.0.0" + } +} diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 98973e948..be89bee8b 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -25,11 +25,12 @@ const initLightning = () => { lightning = createLightningServer({ port: 9999 }); }; -const initWorker = async () => { +const initWorker = async (engineArgs = {}) => { engine = await createEngine({ // logger: createLogger('engine', { level: 'debug' }), logger: createMockLogger(), repoDir: path.resolve('./tmp/repo'), + ...engineArgs, }); worker = createWorkerServer(engine, { @@ -240,7 +241,7 @@ test('run a job with credentials', (t) => { return app.listen(PORT); }; - return new Promise(async (done) => { + return new Promise(async (done) => { const server = createServer(); const config = { username: 'logan', @@ -351,3 +352,40 @@ test.todo('return some kind of error on compilation error'); // }); // }); // }); + + +// set repodir to use the dummy repo +test.only('stateful adaptor should create a new client for each job', (t) => { + return new Promise(async (done) => { + const engineArgs = { + repoDir: path.resolve('./dummy-repo'), + // Important to ensure a single worker. Is there any way I can verify this is is working? + // the job should export a thread id, so let's try that + maxWorkers: 1 + } + + const attempt = { + id: crypto.randomUUID(), + jobs: [ + { + adaptor: '@openfn/stateful-test@1.0.0', + body: `fn(() => { + return { threadId, clientId } + })`, + }, + ], + }; + + initLightning() + + lightning.waitForResult(attempt.id, (result) => { + console.log(result) + t.pass() + done() + }) + + await initWorker(engineArgs); + + lightning.enqueueAttempt(attempt); + }) +}) \ No newline at end of file From 8e1d9e0ddca3c74dc53bb9093a62b63c1e088a62 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 24 Oct 2023 14:43:17 +0200 Subject: [PATCH 207/232] worker-tests: get test working (and failing sadly) --- .../worker/test/integration.test.ts | 49 +- pnpm-lock.yaml | 513 +----------------- 2 files changed, 54 insertions(+), 508 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index be89bee8b..2dc4556ba 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -354,8 +354,9 @@ test.todo('return some kind of error on compilation error'); // }); -// set repodir to use the dummy repo -test.only('stateful adaptor should create a new client for each job', (t) => { +// This test fails, which is a problem! +// the test module does not re-initialise +test.skip('stateful adaptor should create a new client for each job', (t) => { return new Promise(async (done) => { const engineArgs = { repoDir: path.resolve('./dummy-repo'), @@ -364,28 +365,56 @@ test.only('stateful adaptor should create a new client for each job', (t) => { maxWorkers: 1 } - const attempt = { + const attempt1 = { id: crypto.randomUUID(), jobs: [ { adaptor: '@openfn/stateful-test@1.0.0', - body: `fn(() => { + // manual import shouldn't be needed but its not important enough to fight over + body: `import { fn, threadId, clientId } from '@openfn/stateful-test'; + fn(() => { return { threadId, clientId } })`, }, ], }; + const attempt2 = { + ...attempt1, + id: crypto.randomUUID(), + } + let results = {} initLightning() - lightning.waitForResult(attempt.id, (result) => { - console.log(result) - t.pass() - done() - }) + lightning.on('attempt:complete', (evt) => { + const id = evt.attemptId; + results[id] = lightning.getResult(id); + + if (id === attempt2.id) { + const one = results[attempt1.id] + const two = results[attempt2.id] + + t.is(one.threadId, two.threadId); + + // Bugger! The two jobs shared a client + // That's bad, init + t.not(one.clientId, two.clientId); + + done(); + } + }); + + // Note that this API doesn't work!! + // shaeme, it would be useful + // lightning.waitForResult(attempt.id, (result) => { + // console.log(result) + // t.pass() + // done() + // }) await initWorker(engineArgs); - lightning.enqueueAttempt(attempt); + lightning.enqueueAttempt(attempt1); + lightning.enqueueAttempt(attempt2); }) }) \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ddef27cf8..b7ad0667a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,12 +104,6 @@ importers: specifier: ^3.0.2 version: 3.0.2 - integration-tests/cli/repo: - dependencies: - '@openfn/language-common_1.11.1': - specifier: npm:@openfn/language-common@^1.11.1 - version: /@openfn/language-common@1.11.1 - integration-tests/worker: dependencies: '@openfn/engine-multi': @@ -152,17 +146,11 @@ importers: specifier: ^5.1.6 version: 5.1.6 - integration-tests/worker/tmp/repo: + integration-tests/worker/dummy-repo: dependencies: - '@openfn/language-common_latest': - specifier: npm:@openfn/language-common@^1.11.1 - version: /@openfn/language-common@1.11.1 - '@openfn/language-http_latest': - specifier: npm:@openfn/language-http@^5.0.4 - version: /@openfn/language-http@5.0.4 - lodash_latest: - specifier: npm:lodash@^4.17.21 - version: /lodash@4.17.21 + '@openfn/stateful-test_1.0.0': + specifier: '@npm:@openfn/stateful-test@1.0.0' + version: link:@npm:@openfn/stateful-test@1.0.0 packages/cli: dependencies: @@ -442,10 +430,6 @@ importers: specifier: ^5.1.6 version: 5.1.6 - packages/engine-multi/tmp/a/b/c: {} - - packages/engine-multi/tmp/repo: {} - packages/lightning-mock: dependencies: '@koa/router': @@ -1372,11 +1356,6 @@ packages: heap: 0.2.7 dev: false - /@fastify/busboy@2.0.0: - resolution: {integrity: sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==} - engines: {node: '>=14'} - dev: false - /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1642,21 +1621,6 @@ packages: semver: 7.5.4 dev: true - /@openfn/language-common@1.11.1: - resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==} - dependencies: - ajv: 8.12.0 - axios: 1.1.3 - csv-parse: 5.5.2 - csvtojson: 2.0.10 - date-fns: 2.30.0 - jsonpath-plus: 4.0.0 - lodash: 4.17.21 - undici: 5.26.3 - transitivePeerDependencies: - - debug - dev: false - /@openfn/language-common@1.7.5: resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==} dependencies: @@ -1672,22 +1636,6 @@ packages: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] - /@openfn/language-http@5.0.4: - resolution: {integrity: sha512-zuMlJyORxBps0KO+93a3kVBRzStGwYVNAOEl7GgvO6Z96YBO7/K2NiqSyMHTyQeIyCLUBGmrEv1AAuk4pXxKAg==} - dependencies: - '@openfn/language-common': 1.11.1 - cheerio: 1.0.0-rc.12 - cheerio-tableparser: 1.0.1 - csv-parse: 4.16.3 - fast-safe-stringify: 2.1.1 - form-data: 3.0.1 - lodash: 4.17.21 - request: 2.88.2 - tough-cookie: 4.1.3 - transitivePeerDependencies: - - debug - dev: false - /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -2140,24 +2088,6 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 - /ajv@6.12.6: - resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - dependencies: - fast-deep-equal: 3.1.3 - fast-json-stable-stringify: 2.1.0 - json-schema-traverse: 0.4.1 - uri-js: 4.4.1 - dev: false - - /ajv@8.12.0: - resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} - dependencies: - fast-deep-equal: 3.1.3 - json-schema-traverse: 1.0.0 - require-from-string: 2.0.2 - uri-js: 4.4.1 - dev: false - /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -2293,17 +2223,6 @@ packages: resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} engines: {node: '>=12'} - /asn1@0.2.6: - resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} - dependencies: - safer-buffer: 2.1.2 - dev: false - - /assert-plus@1.0.0: - resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} - engines: {node: '>=0.8'} - dev: false - /assign-symbols@1.0.0: resolution: {integrity: sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==} engines: {node: '>=0.10.0'} @@ -2329,6 +2248,7 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -2459,14 +2379,6 @@ packages: fast-glob: 3.3.1 dev: true - /aws-sign2@0.7.0: - resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} - dev: false - - /aws4@1.12.0: - resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} - dev: false - /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: @@ -2484,6 +2396,7 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: true /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -2520,12 +2433,6 @@ packages: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} dev: true - /bcrypt-pbkdf@1.0.2: - resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} - dependencies: - tweetnacl: 0.14.5 - dev: false - /bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: true @@ -2570,17 +2477,9 @@ packages: readable-stream: 4.2.0 dev: true - /bluebird@3.7.2: - resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} - dev: false - /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} - /boolbase@1.0.0: - resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - dev: false - /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2765,10 +2664,6 @@ packages: engines: {node: '>=6'} dev: true - /caseless@0.12.0: - resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} - dev: false - /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -2802,34 +2697,6 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} - /cheerio-select@2.1.0: - resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - dependencies: - boolbase: 1.0.0 - css-select: 5.1.0 - css-what: 6.1.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - dev: false - - /cheerio-tableparser@1.0.1: - resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} - dev: false - - /cheerio@1.0.0-rc.12: - resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} - engines: {node: '>= 6'} - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.1.0 - htmlparser2: 8.0.2 - parse5: 7.1.2 - parse5-htmlparser2-tree-adapter: 7.0.0 - dev: false - /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -3006,6 +2873,7 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 + dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -3080,10 +2948,6 @@ packages: resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} dev: false - /core-util-is@1.0.2: - resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - dev: false - /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -3134,21 +2998,6 @@ packages: which: 2.0.2 dev: true - /css-select@5.1.0: - resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} - dependencies: - boolbase: 1.0.0 - css-what: 6.1.0 - domhandler: 5.0.3 - domutils: 3.1.0 - nth-check: 2.1.1 - dev: false - - /css-what@6.1.0: - resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} - engines: {node: '>= 6'} - dev: false - /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -3165,10 +3014,7 @@ packages: /csv-parse@4.16.3: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} - - /csv-parse@5.5.2: - resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} - dev: false + dev: true /csv-stringify@5.6.5: resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==} @@ -3184,29 +3030,12 @@ packages: stream-transform: 2.1.3 dev: true - /csvtojson@2.0.10: - resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} - engines: {node: '>=4.0.0'} - hasBin: true - dependencies: - bluebird: 3.7.2 - lodash: 4.17.21 - strip-bom: 2.0.0 - dev: false - /currently-unhandled@0.4.1: resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==} engines: {node: '>=0.10.0'} dependencies: array-find-index: 1.0.2 - /dashdash@1.14.1: - resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} - engines: {node: '>=0.10'} - dependencies: - assert-plus: 1.0.0 - dev: false - /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -3341,6 +3170,7 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3391,33 +3221,6 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true - /dom-serializer@2.0.0: - resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - entities: 4.5.0 - dev: false - - /domelementtype@2.3.0: - resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} - dev: false - - /domhandler@5.0.3: - resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} - engines: {node: '>= 4'} - dependencies: - domelementtype: 2.3.0 - dev: false - - /domutils@3.1.0: - resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} - dependencies: - dom-serializer: 2.0.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - dev: false - /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3441,13 +3244,6 @@ packages: /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - /ecc-jsbn@0.1.2: - resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} - dependencies: - jsbn: 0.1.1 - safer-buffer: 2.1.2 - dev: false - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3486,11 +3282,6 @@ packages: ansi-colors: 4.1.3 dev: true - /entities@4.5.0: - resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} - engines: {node: '>=0.12'} - dev: false - /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -4177,10 +3968,6 @@ packages: is-extendable: 1.0.1 dev: true - /extend@3.0.2: - resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - dev: false - /extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} dev: true @@ -4209,15 +3996,6 @@ packages: - supports-color dev: true - /extsprintf@1.3.0: - resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} - engines: {'0': node >=0.6.0} - dev: false - - /fast-deep-equal@3.1.3: - resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - dev: false - /fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} @@ -4250,10 +4028,6 @@ packages: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} dev: false - /fast-json-stable-stringify@2.1.0: - resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} - dev: false - /fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} dev: true @@ -4378,6 +4152,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4392,19 +4167,6 @@ packages: signal-exit: 4.0.2 dev: true - /forever-agent@0.6.1: - resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} - dev: false - - /form-data@2.3.3: - resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} - engines: {node: '>= 0.12'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -4414,15 +4176,6 @@ packages: mime-types: 2.1.35 dev: true - /form-data@3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4430,6 +4183,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -4546,12 +4300,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /getpass@0.1.7: - resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} - dependencies: - assert-plus: 1.0.0 - dev: false - /glob-parent@3.1.0: resolution: {integrity: sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==} dependencies: @@ -4658,20 +4406,6 @@ packages: through2: 2.0.5 dev: true - /har-schema@2.0.0: - resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} - engines: {node: '>=4'} - dev: false - - /har-validator@5.1.5: - resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} - engines: {node: '>=6'} - deprecated: this library is no longer supported - dependencies: - ajv: 6.12.6 - har-schema: 2.0.0 - dev: false - /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -4757,15 +4491,6 @@ packages: lru-cache: 7.18.3 dev: true - /htmlparser2@8.0.2: - resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - entities: 4.5.0 - dev: false - /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4834,15 +4559,6 @@ packages: - supports-color dev: true - /http-signature@1.2.0: - resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} - engines: {node: '>=0.8', npm: '>=1.3.7'} - dependencies: - assert-plus: 1.0.0 - jsprim: 1.4.2 - sshpk: 1.18.0 - dev: false - /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -5236,10 +4952,6 @@ packages: has-symbols: 1.0.3 dev: true - /is-typedarray@1.0.0: - resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} - dev: false - /is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -5249,10 +4961,6 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} - /is-utf8@0.2.1: - resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} - dev: false - /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -5293,10 +5001,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /isstream@0.1.2: - resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} - dev: false - /jackspeak@2.2.2: resolution: {integrity: sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==} engines: {node: '>=14'} @@ -5337,10 +5041,6 @@ packages: argparse: 2.0.1 dev: true - /jsbn@0.1.1: - resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} - dev: false - /json-diff@1.0.6: resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} hasBin: true @@ -5354,22 +5054,6 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true - /json-schema-traverse@0.4.1: - resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - dev: false - - /json-schema-traverse@1.0.0: - resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} - dev: false - - /json-schema@0.4.0: - resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} - dev: false - - /json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - dev: false - /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5384,6 +5068,7 @@ packages: /jsonpath-plus@4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} + dev: true /jsonpath@1.1.1: resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} @@ -5393,16 +5078,6 @@ packages: underscore: 1.12.1 dev: true - /jsprim@1.4.2: - resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} - engines: {node: '>=0.6.0'} - dependencies: - assert-plus: 1.0.0 - extsprintf: 1.3.0 - json-schema: 0.4.0 - verror: 1.10.0 - dev: false - /keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} @@ -6143,16 +5818,6 @@ packages: path-key: 3.1.1 dev: true - /nth-check@2.1.1: - resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} - dependencies: - boolbase: 1.0.0 - dev: false - - /oauth-sign@0.9.0: - resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} - dev: false - /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6422,19 +6087,6 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} - /parse5-htmlparser2-tree-adapter@7.0.0: - resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} - dependencies: - domhandler: 5.0.3 - parse5: 7.1.2 - dev: false - - /parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - dependencies: - entities: 4.5.0 - dev: false - /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6517,10 +6169,6 @@ packages: through2: 2.0.5 dev: true - /performance-now@2.1.0: - resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} - dev: false - /phoenix@1.7.7: resolution: {integrity: sha512-moAN6e4Z16x/x1nswUpnTR2v5gm7HsI7eluZ2YnYUUsBNzi3cY/5frmiJfXIEi877IQAafzTfp8hd6vEUMme+w==} dev: false @@ -6745,6 +6393,7 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -6755,10 +6404,6 @@ packages: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} dev: true - /psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - dev: false - /pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -6781,6 +6426,7 @@ packages: /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} + dev: true /qs@6.11.2: resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} @@ -6789,11 +6435,6 @@ packages: side-channel: 1.0.4 dev: false - /qs@6.5.3: - resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} - engines: {node: '>=0.6'} - dev: false - /query-string@8.1.0: resolution: {integrity: sha512-BFQeWxJOZxZGix7y+SByG3F36dA0AbTy9o6pSmKFcFz7DAj0re9Frkty3saBn3nHo3D0oZJ/+rx3r8H8r8Jbpw==} engines: {node: '>=14.16'} @@ -6803,10 +6444,6 @@ packages: split-on-first: 3.0.0 dev: true - /querystringify@2.2.0: - resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} - dev: false - /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6991,50 +6628,14 @@ packages: engines: {node: '>=0.10'} dev: true - /request@2.88.2: - resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} - engines: {node: '>= 6'} - deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 - dependencies: - aws-sign2: 0.7.0 - aws4: 1.12.0 - caseless: 0.12.0 - combined-stream: 1.0.8 - extend: 3.0.2 - forever-agent: 0.6.1 - form-data: 2.3.3 - har-validator: 5.1.5 - http-signature: 1.2.0 - is-typedarray: 1.0.0 - isstream: 0.1.2 - json-stringify-safe: 5.0.1 - mime-types: 2.1.35 - oauth-sign: 0.9.0 - performance-now: 2.1.0 - qs: 6.5.3 - safe-buffer: 5.2.1 - tough-cookie: 2.5.0 - tunnel-agent: 0.6.0 - uuid: 3.4.0 - dev: false - /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} - /require-from-string@2.0.2: - resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} - engines: {node: '>=0.10.0'} - dev: false - /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true - /requires-port@1.0.0: - resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} - dev: false - /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -7458,22 +7059,6 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - /sshpk@1.18.0: - resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} - engines: {node: '>=0.10.0'} - hasBin: true - dependencies: - asn1: 0.2.6 - assert-plus: 1.0.0 - bcrypt-pbkdf: 1.0.2 - dashdash: 1.14.1 - ecc-jsbn: 0.1.2 - getpass: 0.1.7 - jsbn: 0.1.1 - safer-buffer: 2.1.2 - tweetnacl: 0.14.5 - dev: false - /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7582,13 +7167,6 @@ packages: dependencies: ansi-regex: 6.0.1 - /strip-bom@2.0.0: - resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} - engines: {node: '>=0.10.0'} - dependencies: - is-utf8: 0.2.1 - dev: false - /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -7805,24 +7383,6 @@ packages: nopt: 1.0.10 dev: true - /tough-cookie@2.5.0: - resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} - engines: {node: '>=0.8'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - dev: false - - /tough-cookie@4.1.3: - resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} - engines: {node: '>=6'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - universalify: 0.2.0 - url-parse: 1.5.10 - dev: false - /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false @@ -8082,16 +7642,6 @@ packages: yargs: 17.7.2 dev: true - /tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - dependencies: - safe-buffer: 5.2.1 - dev: false - - /tweetnacl@0.14.5: - resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} - dev: false - /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -8170,13 +7720,6 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true - /undici@5.26.3: - resolution: {integrity: sha512-H7n2zmKEWgOllKkIUkLvFmsJQj062lSm3uA4EYApG8gLuiOM0/go9bIoC3HVaSnfg4xunowDE2i9p8drkXuvDw==} - engines: {node: '>=14.0'} - dependencies: - '@fastify/busboy': 2.0.0 - dev: false - /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'} @@ -8206,11 +7749,6 @@ packages: engines: {node: '>= 4.0.0'} dev: true - /universalify@0.2.0: - resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} - engines: {node: '>= 4.0.0'} - dev: false - /unix-crypt-td-js@1.1.4: resolution: {integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==} dev: true @@ -8232,12 +7770,6 @@ packages: engines: {node: '>=4'} dev: true - /uri-js@4.4.1: - resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} - dependencies: - punycode: 2.3.0 - dev: false - /urix@0.1.0: resolution: {integrity: sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==} deprecated: Please see https://github.com/lydell/urix#deprecated @@ -8248,13 +7780,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false - /url-parse@1.5.10: - resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - dev: false - /use@3.1.1: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} @@ -8273,6 +7798,7 @@ packages: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. hasBin: true + dev: true /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -8295,15 +7821,6 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - /verror@1.10.0: - resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} - engines: {'0': node >=0.6.0} - dependencies: - assert-plus: 1.0.0 - core-util-is: 1.0.2 - extsprintf: 1.3.0 - dev: false - /wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} dependencies: From 881bf90335770ca977088caf8976540355a3fdc9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 09:35:32 +0200 Subject: [PATCH 208/232] worker: tests to prove that worker threads isolate modules --- .../test/worker/worker-pool.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index 8c05cb769..bcc38a84a 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -205,3 +205,29 @@ test.serial('dynamic imports should share state across runs', async (t) => { const count3 = await pool.exec('incrementDynamic', []); t.is(count3, 3); }); + + +// This is kinda done in the tests above, it's just to setup the next test +test.serial('module scope is shared within a thread', async (t) => { + pool = createDedicatedPool({ maxWorkers: 1 }); + + const result = await Promise.all([ + pool.exec('incrementDynamic', []), + pool.exec('incrementDynamic', []), + pool.exec('incrementDynamic', []), + ]) + + t.deepEqual(result, [1, 2, 3]) +}); + +test.serial('module scope is isolated across threads', async (t) => { + pool = createDedicatedPool({ maxWorkers: 3 }); + + const result = await Promise.all([ + pool.exec('incrementDynamic', []), + pool.exec('incrementDynamic', []), + pool.exec('incrementDynamic', []), + ]) + + t.deepEqual(result, [1,1,1]) +}); From 3e378c0a70260fdb9571819203a75ccddbec79f2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 11:20:10 +0200 Subject: [PATCH 209/232] engine: experiment with terminating threads. Didn't work. --- packages/engine-multi/src/api/call-worker.ts | 13 +++++++++-- .../test/worker/worker-pool.test.ts | 22 +++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 6ba6588b8..bf097be3f 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -24,13 +24,22 @@ export default function initWorkers( // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path const workers = createWorkers(workerPath, options); - api.callWorker = (task: string, args: any[] = [], events: any = {}) => - workers.exec(task, args, { + api.callWorker = (task: string, args: any[] = [], events: any = {}) => { + const promise = workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { // just call the callback events[type]?.(args); }, }); + // Tinfoil hat for https://github.com/OpenFn/kit/issues/410 + // Calling this after every job starts will terminate the worker thread AFTER + // the job has executed + // This ensures, for now, that each workflow runs in a clean context + // will this stop jobs returning? Surely not...? + // Will it stop pending jobs running? yes + workers.terminate() + return promise; + } api.closeWorkers = () => workers.terminate(); } diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index bcc38a84a..1ece0b4fd 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -231,3 +231,25 @@ test.serial('module scope is isolated across threads', async (t) => { t.deepEqual(result, [1,1,1]) }); + +// hmm, problematic +test.serial.only('a terminated pool will not return', async (t) => { + return new Promise(done => { + pool = createDedicatedPool({ maxWorkers: 1 }); + + pool.exec('incrementDynamic', []).then(result => { + console.log('result', result) + }).catch(e => { + console.log('error', e) + + }) + pool.terminate() +}) + + // const result = await Promise.all([ + // pool.exec('incrementDynamic', []), + // pool.exec('incrementDynamic', []), + // ]) + + // t.deepEqual(result, [1, 2, 3]) +}) \ No newline at end of file From 463169956c66c2eecc59a8a09084d3ee4901298b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 11:51:05 +0200 Subject: [PATCH 210/232] runtime: support a cacheKey in linker opts --- packages/runtime/README.md | 14 ++++++ packages/runtime/src/modules/linker.ts | 13 ++++- .../runtime/test/__modules__/number-export.js | 13 ++++- packages/runtime/test/modules/linker.test.ts | 48 +++++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/packages/runtime/README.md b/packages/runtime/README.md index 53c294a5b..52723b6cd 100644 --- a/packages/runtime/README.md +++ b/packages/runtime/README.md @@ -38,6 +38,20 @@ The runtime provides no CLI. Use packages/cli (devtools) for this. For the runtime to work, the parent process needs `--experimental-vm-modules` be passed. You may also want to pass `--no-warnings` to suppress annoying console warnings. +## Module Caching + +If running in a long-lived process (such as inside ws-worker), the runtime may import cached modules. + +This can be a problem for isolation even within the sandbox, because state can be shared by two workflows using the same adaptor. This is a security and stability concern. + +To address this, the runtime accepts a cacheKey on the linker options. If set, this will be appended to the linker's imports (ie, top-level job imports). All jobs in the same workflow will use the same cacheKey, so a module is cached between jobs, but NOT between workflows. + +Long-running worker processes should pass a unique cache key with each run. + +IMPORTANT: This will leak memory, because loaded but "stale" modules will NOT be garbage collected. + +It is expected that that long-running runtimes will have some kind of purge functionality to reclaim memory (for example, engine-multi will regulaly burn worker threads) + ## Execution Plans The runtime can accept an Execution Plan (or workflow) as an input. diff --git a/packages/runtime/src/modules/linker.ts b/packages/runtime/src/modules/linker.ts index a85f35f62..318db5c1a 100644 --- a/packages/runtime/src/modules/linker.ts +++ b/packages/runtime/src/modules/linker.ts @@ -25,6 +25,11 @@ export type LinkerOptions = { repo?: string; whitelist?: RegExp[]; // whitelist packages which the linker allows to be imported + + // use this to add a cache-busting id to all imports + // Used in long-running processes to ensure that each job has an isolated + // top module scope for all imports + cacheKey?: string; }; export type Linker = ( @@ -91,7 +96,10 @@ const loadActualModule = async (specifier: string, options: LinkerOptions) => { // If the specifier is a path, just import it if (specifier.startsWith('/') && specifier.endsWith('.js')) { - const importPath = `${prefix}${specifier}`; + let importPath = `${prefix}${specifier}`; + if (options.cacheKey) { + importPath += '?cache=' + options.cacheKey; + } log.debug(`[linker] Loading module from path: ${importPath}`); return import(importPath); } @@ -125,6 +133,9 @@ const loadActualModule = async (specifier: string, options: LinkerOptions) => { if (path) { log.debug(`[linker] Loading module ${specifier} from ${path}`); try { + if (options.cacheKey) { + path += '?cache=' + options.cacheKey; + } const result = import(`${prefix}${path}`); if (specifier.startsWith('@openfn/language-')) { log.info(`Resolved adaptor ${specifier} to version ${version}`); diff --git a/packages/runtime/test/__modules__/number-export.js b/packages/runtime/test/__modules__/number-export.js index 15d162472..b4cc0f7dc 100644 --- a/packages/runtime/test/__modules__/number-export.js +++ b/packages/runtime/test/__modules__/number-export.js @@ -1 +1,12 @@ -export default 20; +let number = 20; + +export default number; + +export function increment() { + return ++number; +} + +export function getNumber() { + return number; +} + diff --git a/packages/runtime/test/modules/linker.test.ts b/packages/runtime/test/modules/linker.test.ts index 6a76f1c9c..a07a198f4 100644 --- a/packages/runtime/test/modules/linker.test.ts +++ b/packages/runtime/test/modules/linker.test.ts @@ -105,6 +105,54 @@ test('loads a module from a path', async (t) => { t.assert(m.namespace.default === 'test'); }); +test('imports with a cacheKey', async (t) => { + const opts = { + ...options, + cacheKey: 'abc' + }; + const m = await linker('ultimate-answer', context, opts); + t.assert(m.namespace.default === 43); +}); + +test('modules will be cached by default', async (t) => { + const modulePath = path.resolve('test/__modules__/number-export.js'); + const m1 = await linker(modulePath, context, options); + + t.is(m1.namespace.getNumber(), 20); + + const result = m1.namespace.increment() + + t.is(result, 21) + + const m2 = await linker(modulePath, context, options); + + t.is(m2.namespace.getNumber(), 21); +}); + +test('cachekey busts the module cache', async (t) => { + const modulePath = path.resolve('test/__modules__/number-export.js'); + + const opts1 = { + ...options, + cacheKey: 'a' + } + const m1 = await linker(modulePath, context, opts1); + + t.is(m1.namespace.getNumber(), 20); + + const result = m1.namespace.increment() + + t.is(result, 21) + + const opts2 = { + ...options, + cacheKey: 'b' + } + const m2 = await linker(modulePath, context, opts2); + + t.is(m2.namespace.getNumber(), 20); +}); + test('throw if a non-whitelisted value is passed', async (t) => { await t.throwsAsync(() => linker('i-heart-hacking', context, { repo, whitelist: [/^@openfn\//] }) From d2360d4147fe99d47c931a385a3da3a8942503f3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 11:51:55 +0200 Subject: [PATCH 211/232] runtime: changeset --- .changeset/short-pens-punch.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/short-pens-punch.md diff --git a/.changeset/short-pens-punch.md b/.changeset/short-pens-punch.md new file mode 100644 index 000000000..d5363759f --- /dev/null +++ b/.changeset/short-pens-punch.md @@ -0,0 +1,5 @@ +--- +'@openfn/runtime': patch +--- + +Support a cacheKey to bust cached modules in long-running processes From 46865081542635493c8296b2b3b07a6085cd49cd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 12:47:30 +0200 Subject: [PATCH 212/232] worker: use plan id as a cache key --- packages/engine-multi/src/api/call-worker.ts | 14 ++---------- packages/engine-multi/src/worker/worker.ts | 2 +- .../test/worker/worker-pool.test.ts | 22 ------------------- 3 files changed, 3 insertions(+), 35 deletions(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index bf097be3f..fb883b7c5 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -24,22 +24,12 @@ export default function initWorkers( // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path const workers = createWorkers(workerPath, options); - api.callWorker = (task: string, args: any[] = [], events: any = {}) => { - const promise = workers.exec(task, args, { + api.callWorker = (task: string, args: any[] = [], events: any = {}) => workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { // just call the callback events[type]?.(args); }, - }); - // Tinfoil hat for https://github.com/OpenFn/kit/issues/410 - // Calling this after every job starts will terminate the worker thread AFTER - // the job has executed - // This ensures, for now, that each workflow runs in a clean context - // will this stop jobs returning? Surely not...? - // Will it stop pending jobs running? yes - workers.terminate() - return promise; - } + }) api.closeWorkers = () => workers.terminate(); } diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts index 42da9edc7..69207aa0e 100644 --- a/packages/engine-multi/src/worker/worker.ts +++ b/packages/engine-multi/src/worker/worker.ts @@ -31,13 +31,13 @@ workerpool.worker({ run: (plan: ExecutionPlan, runOptions: RunOptions) => { const { adaptorPaths, whitelist, sanitize } = runOptions; const { logger, jobLogger } = createLoggers(plan.id!, sanitize); - const options = { logger, jobLogger, linker: { modules: adaptorPaths, whitelist, + cacheKey: plan.id, }, callbacks: { // TODO: this won't actually work across the worker boundary diff --git a/packages/engine-multi/test/worker/worker-pool.test.ts b/packages/engine-multi/test/worker/worker-pool.test.ts index 1ece0b4fd..bcc38a84a 100644 --- a/packages/engine-multi/test/worker/worker-pool.test.ts +++ b/packages/engine-multi/test/worker/worker-pool.test.ts @@ -231,25 +231,3 @@ test.serial('module scope is isolated across threads', async (t) => { t.deepEqual(result, [1,1,1]) }); - -// hmm, problematic -test.serial.only('a terminated pool will not return', async (t) => { - return new Promise(done => { - pool = createDedicatedPool({ maxWorkers: 1 }); - - pool.exec('incrementDynamic', []).then(result => { - console.log('result', result) - }).catch(e => { - console.log('error', e) - - }) - pool.terminate() -}) - - // const result = await Promise.all([ - // pool.exec('incrementDynamic', []), - // pool.exec('incrementDynamic', []), - // ]) - - // t.deepEqual(result, [1, 2, 3]) -}) \ No newline at end of file From 107ed756304c28567a03ed11a7f525798bc39808 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 27 Oct 2023 12:48:06 +0200 Subject: [PATCH 213/232] worker-tests: add stateful adaptor test tests are failing but unrelated --- integration-tests/worker/test/integration.test.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 2dc4556ba..095e1566d 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -354,14 +354,10 @@ test.todo('return some kind of error on compilation error'); // }); -// This test fails, which is a problem! -// the test module does not re-initialise -test.skip('stateful adaptor should create a new client for each job', (t) => { +test.only('stateful adaptor should create a new client for each attempt', (t) => { return new Promise(async (done) => { const engineArgs = { repoDir: path.resolve('./dummy-repo'), - // Important to ensure a single worker. Is there any way I can verify this is is working? - // the job should export a thread id, so let's try that maxWorkers: 1 } @@ -393,11 +389,9 @@ test.skip('stateful adaptor should create a new client for each job', (t) => { if (id === attempt2.id) { const one = results[attempt1.id] const two = results[attempt2.id] - + // The module should be isolated within the same thread t.is(one.threadId, two.threadId); - // Bugger! The two jobs shared a client - // That's bad, init t.not(one.clientId, two.clientId); done(); From 1117ec825193d476ee2b47c4ea393cace10f39fd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 11:31:05 +0000 Subject: [PATCH 214/232] engine: support timeout on the attempt --- packages/engine-multi/src/api.ts | 5 --- packages/engine-multi/src/api/call-worker.ts | 18 +++++++++-- packages/engine-multi/src/api/execute.ts | 13 ++++++-- packages/engine-multi/src/engine.ts | 5 +++ packages/engine-multi/src/errors.ts | 5 +++ .../engine-multi/src/test/worker-functions.js | 5 ++- packages/engine-multi/src/types.ts | 3 +- packages/engine-multi/test/api.test.ts | 4 +-- .../engine-multi/test/api/call-worker.test.ts | 8 ++++- .../engine-multi/test/api/execute.test.ts | 32 ++++++++++++++++++- packages/engine-multi/test/engine.test.ts | 30 +++++++++++++++++ 11 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 packages/engine-multi/src/errors.ts diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index 8d0bf95a1..a9cb69255 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -43,11 +43,6 @@ const createAPI = async function (options: RTEOptions = {}) { logger.warn('Set env var ENGINE_REPO_DIR to use a different directory'); } } - - // re logging, for example, where does this go? - // it's not an attempt log - // it probably shouldnt be sent to the worker - // but it is an important bit of debugging logger.info('repoDir set to ', repoDir); const engineOptions = { diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index fb883b7c5..047037715 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -13,6 +13,7 @@ type WorkerOptions = { minWorkers?: number; maxWorkers?: number; env?: any; + timeout?: number; // ms }; // Adds a `callWorker` function to the API object, which will execute a task in a worker @@ -24,12 +25,25 @@ export default function initWorkers( // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path const workers = createWorkers(workerPath, options); - api.callWorker = (task: string, args: any[] = [], events: any = {}) => workers.exec(task, args, { + api.callWorker = ( + task: string, + args: any[] = [], + events: any = {}, + timeout?: number + ) => { + const promise = workers.exec(task, args, { on: ({ type, ...args }: WorkerEvent) => { // just call the callback events[type]?.(args); }, - }) + }); + + if (timeout) { + promise.timeout(timeout); + } + + return promise; + }; api.closeWorkers = () => workers.terminate(); } diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 182877989..0d1b7283c 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -16,7 +16,6 @@ import preloadCredentials from './preload-credentials'; const execute = async (context: ExecutionContext) => { const { state, callWorker, logger, options } = context; - const adaptorPaths = await autoinstall(context); await compile(context); @@ -52,12 +51,20 @@ const execute = async (context: ExecutionContext) => { log(context, evt); }, }; - return callWorker('run', [state.plan, runOptions], events).catch((e: any) => { + return callWorker( + 'run', + [state.plan, runOptions], + events, + options.timeout + ).catch((e: any) => { + // E could be: + // A timeout + // An crash error within the job + // TODO what information can I usefully provide here? // DO I know which job I'm on? // DO I know the thread id? // Do I know where the error came from? - error(context, { workflowId: state.plan.id, error: e }); logger.error(e); }); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index cf89369b8..e81bf1dd7 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -76,11 +76,15 @@ export type EngineOptions = { maxWorkers?: number; whitelist?: RegExp[]; + + // Timeout for the whole workflow + timeout?: number; }; export type ExecuteOptions = { sanitize?: SanitizePolicies; resolvers?: LazyResolvers; + timeout?: number; }; // This creates the internal API @@ -159,6 +163,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { ...options, sanitize: opts.sanitize, resolvers: opts.resolvers, + timeout: opts.timeout, }, }); diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts new file mode 100644 index 000000000..708bbf736 --- /dev/null +++ b/packages/engine-multi/src/errors.ts @@ -0,0 +1,5 @@ +// This will eventually contain all the error classes thrown by the engine +import { Promise as WorkerPoolPromise } from 'workerpool'; + +// Thrown when the whole workflow is timedout +export const TimeoutError = WorkerPoolPromise.TimeoutError; diff --git a/packages/engine-multi/src/test/worker-functions.js b/packages/engine-multi/src/test/worker-functions.js index 283a8c80b..0a10c4fe9 100644 --- a/packages/engine-multi/src/test/worker-functions.js +++ b/packages/engine-multi/src/test/worker-functions.js @@ -37,7 +37,6 @@ workerpool.worker({ try { const [job] = plan.jobs; const result = eval(job.expression); - workerpool.workerEmit({ type: 'worker:workflow-complete', workflowId, @@ -58,6 +57,10 @@ workerpool.worker({ } }, + timeout: () => { + while (true) {} + }, + // Experiments with freezing the global scope // We may do this in the actual worker freeze: () => { diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index 55b349157..2b99f0c78 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -33,7 +33,8 @@ export type WorkflowState = { export type CallWorker = ( task: string, args: any[], - events?: any + events?: any, + timeout?: number ) => workerpool.Promise; export type ExecutionContextConstructor = { diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index e9d2de120..a7f1f2825 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -5,7 +5,7 @@ import { createMockLogger } from '@openfn/logger'; // thes are tests on the public api functions generally // so these are very high level tests and don't allow mock workers or anything -const logger = createMockLogger(); +const logger = createMockLogger(undefined, { level: 'debug' }); test.afterEach(() => { logger._reset(); @@ -19,7 +19,7 @@ test.serial('create a default engine api without throwing', async (t) => { test.serial('create an engine api with options without throwing', async (t) => { await createAPI({ logger }); // just a token test to see if the logger is accepted and used - t.assert(logger._history.length > 0); + t.true(logger._history.length > 0); }); test.serial('create an engine api with a limited surface', async (t) => { diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index b864e6da5..aba5b868f 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -1,6 +1,6 @@ import test from 'ava'; import path from 'node:path'; - +import { Promise as WorkerPoolPromise } from 'workerpool'; import initWorkers, { createWorkers } from '../../src/api/call-worker'; import { EngineAPI } from '../../src/types'; @@ -41,6 +41,12 @@ test('callWorker should trigger an event callback', async (t) => { }); }); +test('callWorker should throw TimeoutError if it times out', async (t) => { + await t.throwsAsync(() => api.callWorker('timeout', [11], {}, 10), { + instanceOf: WorkerPoolPromise.TimeoutError, + }); +}); + // Dang, this doesn't work, the worker threads run in the same process test.skip('callWorker should execute with a different process id', async (t) => { return new Promise((done) => { diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index a08d0b411..4bad58dee 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -1,6 +1,5 @@ import path from 'node:path'; import test from 'ava'; -import { EventEmitter } from 'node:events'; import { WorkflowState } from '../../src/types'; import initWorkers from '../../src/api/call-worker'; import execute from '../../src/api/execute'; @@ -9,6 +8,7 @@ import { JOB_COMPLETE, JOB_START, WORKFLOW_COMPLETE, + WORKFLOW_ERROR, WORKFLOW_LOG, WORKFLOW_START, } from '../../src/events'; @@ -155,6 +155,36 @@ test.serial('should emit a log event', async (t) => { t.is(workflowLog.level, 'info'); }); +test.serial('should emit error on timeout', async (t) => { + const state = { + id: 'zz', + plan: { + jobs: [ + { + expression: '() => { while(true) {} }', + }, + ], + }, + } as WorkflowState; + + const wfOptions = { + ...options, + timeout: 10, + }; + + let event; + + const context = createContext({ state, options: wfOptions }); + + context.once(WORKFLOW_ERROR, (evt) => (event = evt)); + + await execute(context); + + t.truthy(event.threadId); + t.is(event.type, 'TimeoutError'); + t.assert(event.message.match(/Promise timed out after/)); +}); + // how will we test compilation? // compile will call the actual runtime // maybe that's fine? diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 10da32cc8..b0592d65a 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -203,3 +203,33 @@ test('catch and emit errors', async (t) => { }); }); }); + +test('timeout the whole attempt and emit an error', async (t) => { + return new Promise(async (done) => { + const p = path.resolve('src/test/worker-functions.js'); + const engine = await createEngine(options, p); + + const plan = { + id: 'a', + jobs: [ + { + expression: 'while(true) {}', + }, + ], + }; + + const opts = { + timeout: 10, + }; + + engine.execute(plan, opts); + + engine.listen(plan.id, { + [e.WORKFLOW_ERROR]: ({ message, type }) => { + t.is(type, 'TimeoutError'); + t.is(message, 'Promise timed out after 10 ms'); + done(); + }, + }); + }); +}); From a6d53185267c80bc503eef60d4d982ab20a43c98 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 11:42:57 +0000 Subject: [PATCH 215/232] worker-tests: restore tests --- integration-tests/worker/package.json | 3 +- .../worker/test/integration.test.ts | 29 +++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 177c07c80..83d8c6bfa 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -11,8 +11,7 @@ "build:pack": "pnpm clean && cd ../.. && pnpm pack:local integration-tests/worker/dist --no-version", "build": "pnpm build:pack && docker build --tag worker-integration-tests .", "start": "docker run worker-integration-tests", - "_test": "pnpm clean && npx ava -s --timeout 2m", - "test": "npx ava -s --timeout 2m" + "test": "pnpm clean && npx ava -s --timeout 2m" }, "dependencies": { "@openfn/engine-multi": "workspace:^", diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts index 095e1566d..091b4ea7a 100644 --- a/integration-tests/worker/test/integration.test.ts +++ b/integration-tests/worker/test/integration.test.ts @@ -69,6 +69,7 @@ test('should join attempts queue channel', (t) => { test('should run a simple job with no compilation or adaptor', (t) => { return new Promise(async (done) => { initLightning(); + lightning.on('attempt:complete', (evt) => { // This will fetch the final dataclip from the attempt const result = lightning.getResult('a1'); @@ -77,6 +78,7 @@ test('should run a simple job with no compilation or adaptor', (t) => { t.pass('completed attempt'); done(); }); + await initWorker(); lightning.enqueueAttempt({ @@ -91,8 +93,6 @@ test('should run a simple job with no compilation or adaptor', (t) => { }); }); -// todo ensure repo is clean -// check how we manage the env in cli tests test('run a job with autoinstall of common', (t) => { return new Promise(async (done) => { initLightning(); @@ -353,13 +353,12 @@ test.todo('return some kind of error on compilation error'); // }); // }); - -test.only('stateful adaptor should create a new client for each attempt', (t) => { +test('stateful adaptor should create a new client for each attempt', (t) => { return new Promise(async (done) => { const engineArgs = { repoDir: path.resolve('./dummy-repo'), - maxWorkers: 1 - } + maxWorkers: 1, + }; const attempt1 = { id: crypto.randomUUID(), @@ -377,18 +376,18 @@ test.only('stateful adaptor should create a new client for each attempt', (t) => const attempt2 = { ...attempt1, id: crypto.randomUUID(), - } - let results = {} + }; + let results = {}; - initLightning() + initLightning(); lightning.on('attempt:complete', (evt) => { const id = evt.attemptId; - results[id] = lightning.getResult(id); + results[id] = lightning.getResult(id); if (id === attempt2.id) { - const one = results[attempt1.id] - const two = results[attempt2.id] + const one = results[attempt1.id]; + const two = results[attempt2.id]; // The module should be isolated within the same thread t.is(one.threadId, two.threadId); @@ -399,7 +398,7 @@ test.only('stateful adaptor should create a new client for each attempt', (t) => }); // Note that this API doesn't work!! - // shaeme, it would be useful + // shaeme, it would be useful // lightning.waitForResult(attempt.id, (result) => { // console.log(result) // t.pass() @@ -410,5 +409,5 @@ test.only('stateful adaptor should create a new client for each attempt', (t) => lightning.enqueueAttempt(attempt1); lightning.enqueueAttempt(attempt2); - }) -}) \ No newline at end of file + }); +}); From e24df80438a4d5a3ad06547cba46828d9c617444 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 11:43:10 +0000 Subject: [PATCH 216/232] runtime: comment --- packages/runtime/src/runtime.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/src/runtime.ts b/packages/runtime/src/runtime.ts index 8743ce20d..8230b21a7 100644 --- a/packages/runtime/src/runtime.ts +++ b/packages/runtime/src/runtime.ts @@ -21,7 +21,7 @@ export type Options = { logger?: Logger; jobLogger?: Logger; - timeout?: number; + timeout?: number; // this is timeout used per job, not per workflow strict?: boolean; // Be strict about handling of state returned from jobs deleteConfiguration?: boolean; From f10fdf7a2ebd0d1b747e9648c962af50a9f58015 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 11:47:44 +0000 Subject: [PATCH 217/232] engine: remove test event --- packages/engine-multi/src/engine.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index e81bf1dd7..c108bb9dd 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -213,8 +213,6 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { // How does this work if deferred? }; - engine.emit('test'); // TODO remove - return Object.assign(engine, { options, workerPath: resolvedWorkerPath, From caab7a7c71e198c3d5ecb911eb5bb8e7cf6d5566 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 12:06:25 +0000 Subject: [PATCH 218/232] engine: try to purge workers after every run --- packages/engine-multi/src/api.ts | 1 + packages/engine-multi/src/api/call-worker.ts | 18 +++++++++-- packages/engine-multi/src/engine.ts | 13 +++++--- packages/engine-multi/src/events.ts | 3 ++ packages/engine-multi/test/api.test.ts | 32 ++++++++++++++++++- .../engine-multi/test/api/call-worker.test.ts | 18 +++++++++-- 6 files changed, 76 insertions(+), 9 deletions(-) diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts index a9cb69255..2f043fe2a 100644 --- a/packages/engine-multi/src/api.ts +++ b/packages/engine-multi/src/api.ts @@ -72,6 +72,7 @@ const createAPI = async function (options: RTEOptions = {}) { return { execute: engine.execute, listen: engine.listen, + on: (evt: string, fn: (...args: any[]) => void) => engine.on(evt, fn), }; }; diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 047037715..29a220246 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -1,7 +1,11 @@ import { fileURLToPath } from 'node:url'; import path from 'node:path'; import workerpool from 'workerpool'; -import { EngineAPI } from '../types'; + +import { PURGE } from '../events'; + +import type { EngineAPI } from '../types'; +import type { Logger } from '@openfn/logger'; // All events coming out of the worker need to include a type key type WorkerEvent = { @@ -20,7 +24,8 @@ type WorkerOptions = { export default function initWorkers( api: EngineAPI, workerPath: string, - options: WorkerOptions = {} + options: WorkerOptions = {}, + logger?: Logger ) { // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path @@ -42,6 +47,15 @@ export default function initWorkers( promise.timeout(timeout); } + promise.then(() => { + const { pendingTasks } = workers.stats(); + if (pendingTasks == 0) { + logger?.debug('Purging workers'); + api.emit(PURGE); + workers.terminate(); + } + }); + return promise; }; diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index c108bb9dd..aff43405f 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -121,10 +121,15 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { const engine = new Engine() as EngineAPI; - initWorkers(engine, resolvedWorkerPath, { - minWorkers: options.minWorkers, - maxWorkers: options.maxWorkers, - }); + initWorkers( + engine, + resolvedWorkerPath, + { + minWorkers: options.minWorkers, + maxWorkers: options.maxWorkers, + }, + options.logger + ); await validateWorker(engine); diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index cba5e699d..cfbc4f433 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -24,6 +24,8 @@ export const AUTOINSTALL_COMPLETE = 'autoinstall-complete'; export const AUTOINSTALL_ERROR = 'autoinstall-error'; +export const PURGE = 'purge-workers'; + export type EventMap = { [WORKFLOW_START]: WorkflowStartPayload; [WORKFLOW_COMPLETE]: WorkflowCompletePayload; @@ -33,6 +35,7 @@ export type EventMap = { [WORKFLOW_ERROR]: WorkflowErrorPayload; [AUTOINSTALL_COMPLETE]: AutoinstallCompletePayload; [AUTOINSTALL_ERROR]: AutoinstallErrorPayload; + [PURGE]: null; }; export type ExternalEvents = keyof EventMap; diff --git a/packages/engine-multi/test/api.test.ts b/packages/engine-multi/test/api.test.ts index a7f1f2825..36a8baf29 100644 --- a/packages/engine-multi/test/api.test.ts +++ b/packages/engine-multi/test/api.test.ts @@ -1,6 +1,7 @@ import test from 'ava'; import createAPI from '../src/api'; import { createMockLogger } from '@openfn/logger'; +import { PURGE } from '../src/events'; // thes are tests on the public api functions generally // so these are very high level tests and don't allow mock workers or anything @@ -27,7 +28,7 @@ test.serial('create an engine api with a limited surface', async (t) => { const keys = Object.keys(api); // TODO the api will actually probably get a bit bigger than this - t.deepEqual(keys, ['execute', 'listen']); + t.deepEqual(keys, ['execute', 'listen', 'on']); }); // Note that this runs with the actual runtime worker @@ -94,3 +95,32 @@ test.serial('should listen to workflow-complete', async (t) => { }); }); }); + +test.serial('should purge workers after a single run', async (t) => { + return new Promise(async (done) => { + const api = await createAPI({ + logger, + // Disable compilation + compile: { + skip: true, + }, + }); + + const plan = { + id: 'a', + jobs: [ + { + expression: 'export default [s => s]', + // with no adaptor it shouldn't try to autoinstall + }, + ], + }; + + api.on(PURGE, () => { + t.pass('workers purged'); + done(); + }); + + api.execute(plan); + }); +}); diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index aba5b868f..74314412c 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -1,10 +1,13 @@ import test from 'ava'; import path from 'node:path'; +import EventEmitter from 'node:events'; import { Promise as WorkerPoolPromise } from 'workerpool'; -import initWorkers, { createWorkers } from '../../src/api/call-worker'; + +import initWorkers from '../../src/api/call-worker'; import { EngineAPI } from '../../src/types'; +import { PURGE } from '../../src/events'; -let api = {} as EngineAPI; +let api = new EventEmitter() as EngineAPI; const workerPath = path.resolve('src/test/worker-functions.js'); @@ -74,6 +77,17 @@ test('callWorker should execute in a different process', async (t) => { }); }); +test('callWorker should try to purge workers on complete', async (t) => { + return new Promise((done) => { + api.on(PURGE, () => { + t.pass('purge event called'); + done(); + }); + + api.callWorker('test', []); + }); +}); + test('If null env is passed, worker thread should be able to access parent env', async (t) => { const badAPI = {} as EngineAPI; const env = null; From bc8f531ff4601fcb30b9dcf7ff75efc9adbc534d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 12:11:31 +0000 Subject: [PATCH 219/232] docs --- packages/engine-multi/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/engine-multi/README.md b/packages/engine-multi/README.md index 0a4f8362f..f0280d455 100644 --- a/packages/engine-multi/README.md +++ b/packages/engine-multi/README.md @@ -93,7 +93,7 @@ The main implementation is in engine.ts, which exposes a much broader interface, When execute is called and passed a plan, the engine first generates an execution context. This contains an event emitter just for that workflower and some contextualised state. -## Security Considerations +## Security Considerations & Memory Management The engine uses workerpool to maintain a pool of worker threads. @@ -101,10 +101,13 @@ As workflows come in to be executed, they are passed to workerpool which will pi workerpool has no natural environment hardening, which means workflows running in the same thread will share an environment. Globals set in workflow A will be available to workflow B, and by the same token an adaptor loaded for workflow A will be shared with workflow B. -We have two mitigations to this: +Also, because the thread is long-lived, modules imported into the sandbox will be shared. + +We have several mitgations against this, ensuring a safe, secure and stable execution environment: - The runtime sandbox itself ensures that each job runs in an isolated context. If a job escapes the sandbox, it will have access to the thread's global scope -- Inside the worker thread, we freeze the global scope. This basically means that jobs are unable to write data to the global scope. +- Each workflow appends a unique id to all its imports, busting the node cache and forcing each module to be re-initialised. This means workers cannot share adaptors and all state is reset. +- To preserve memory, worker threads are regularly purged, meaning destroyed (note that this comes with a performance hit and undermines the use of worker pooling entirely!). When each workflow is complete, if there are no pending tasks to execute, all worker threads are destroyed. Inside the worker thread, we ensure that: From e4a16e68f59cab9b2e6d0daf3b7ee324193cce52 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 12:45:23 +0000 Subject: [PATCH 220/232] engine: prep for release --- packages/engine-multi/CHANGELOG.md | 321 +---------------------------- packages/engine-multi/package.json | 6 +- packages/engine-multi/test.js | 29 --- 3 files changed, 12 insertions(+), 344 deletions(-) delete mode 100644 packages/engine-multi/test.js diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 9a445ad7c..d11bbf1a0 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,316 +1,13 @@ -# runtime-manager +# engine-multi -## 0.0.41 +## 0.1.0 -Testing auto-deploy, no changes +First release of the multi-threaded runtime engine. -## 0.0.40 +Features: -### Patch Changes - -- Updated dependencies [0ff4f98] - - @openfn/runtime@0.0.31 - -## 0.0.39 - -### Patch Changes - -- Updated dependencies - - @openfn/logger@0.0.17 - - @openfn/runtime@0.0.30 - - @openfn/compiler@0.0.36 - -## 0.0.38 - -### Patch Changes - -- Updated dependencies [2a0aaa9] - - @openfn/compiler@0.0.35 - - @openfn/runtime@0.0.29 - - @openfn/logger@0.0.16 - -## 0.0.37 - -### Patch Changes - -- faf1852: Downgrade tsup -- Updated dependencies [faf1852] - - @openfn/compiler@0.0.34 - - @openfn/logger@0.0.15 - - @openfn/runtime@0.0.28 - -## 0.0.36 - -### Patch Changes - -- Updated dependencies [749afe8] -- Updated dependencies [614c86b] -- Updated dependencies [4c875b3] -- Updated dependencies [4c875b3] - - @openfn/logger@0.0.14 - - @openfn/runtime@0.0.27 - - @openfn/compiler@0.0.33 - -## 0.0.35 - -### Patch Changes - -- Updated dependencies - - @openfn/runtime@0.0.26 - -## 0.0.34 - -### Patch Changes - -- Updated dependencies [2024ce8] - - @openfn/runtime@0.0.25 - -## 0.0.33 - -### Patch Changes - -- Updated dependencies [6f51ce2] - - @openfn/logger@0.0.13 - - @openfn/runtime@0.0.24 - - @openfn/compiler@0.0.32 - -## 0.0.32 - -### Patch Changes - -- 91a3311: checked-in package-lock changes for language-common -- Updated dependencies [91a3311] - - @openfn/compiler@0.0.31 - - @openfn/runtime@0.0.23 - -## 0.0.31 - -### Patch Changes - -- Updated dependencies [c341ff0] - - @openfn/compiler@0.0.30 - - @openfn/runtime@0.0.22 - -## 0.0.30 - -### Patch Changes - -- Updated dependencies [7df08d4] - - @openfn/compiler@0.0.29 - -## 0.0.29 - -### Patch Changes - -- Updated dependencies [f4b9702] - - @openfn/compiler@0.0.28 - - @openfn/logger@0.0.12 - - @openfn/runtime@0.0.21 - -## 0.0.28 - -### Patch Changes - -- @openfn/compiler@0.0.27 - -## 0.0.27 - -### Patch Changes - -- Updated dependencies [60f695f] -- Updated dependencies [d67f45a] - - @openfn/runtime@0.0.20 - - @openfn/logger@0.0.11 - - @openfn/compiler@0.0.26 - -## 0.0.26 - -### Patch Changes - -- Updated dependencies [38ad73e] - - @openfn/logger@0.0.10 - - @openfn/compiler@0.0.25 - - @openfn/runtime@0.0.19 - -## 0.0.25 - -### Patch Changes - -- Updated dependencies [e43d3ba] -- Updated dependencies [e43d3ba] - - @openfn/logger@0.0.9 - - @openfn/runtime@0.0.18 - - @openfn/compiler@0.0.24 - -## 0.0.24 - -### Patch Changes - -- Updated dependencies [19e9f31] - - @openfn/runtime@0.0.17 - -## 0.0.23 - -### Patch Changes - -- Updated dependencies - - @openfn/compiler@0.0.23 - - @openfn/runtime@0.0.16 - -## 0.0.22 - -### Patch Changes - -- Updated dependencies [986bf07] -- Updated dependencies [5c6fde4] - - @openfn/runtime@0.0.15 - -## 0.0.21 - -### Patch Changes - -- Updated dependencies [47ac1a9] -- Updated dependencies [1695874] - - @openfn/runtime@0.0.14 - - @openfn/compiler@0.0.22 - -## 0.0.20 - -### Patch Changes - -- Updated dependencies [454a06b] - - @openfn/compiler@0.0.21 - -## 0.0.19 - -### Patch Changes - -- @openfn/compiler@0.0.20 - -## 0.0.18 - -### Patch Changes - -- @openfn/compiler@0.0.19 -- @openfn/runtime@0.0.13 - -## 0.0.17 - -### Patch Changes - -- @openfn/compiler@0.0.18 - -## 0.0.16 - -### Patch Changes - -- @openfn/compiler@0.0.17 -- @openfn/runtime@0.0.12 - -## 0.0.15 - -### Patch Changes - -- @openfn/compiler@0.0.15 - -## 0.0.14 - -### Patch Changes - -- Updated dependencies [ba9bf80] - - @openfn/runtime@0.0.11 - - @openfn/compiler@0.0.14 - -## 0.0.13 - -### Patch Changes - -- @openfn/compiler@0.0.13 - -## 0.0.12 - -### Patch Changes - -- Updated dependencies - - @openfn/runtime@0.0.10 - - @openfn/compiler@0.0.12 - -## 0.0.11 - -### Patch Changes - -- 28168a8: Updated build process -- Updated dependencies [6d1d199] - - @openfn/runtime@0.0.9 - - @openfn/compiler@0.0.11 - -## 0.0.10 - -### Patch Changes - -- Updated dependencies [1d293ae] - - @openfn/compiler@0.0.10 - -## 0.0.9 - -### Patch Changes - -- 92e5427: bump everything, npm package.json issues -- Updated dependencies [92e5427] - - @openfn/compiler@0.0.9 - - @openfn/runtime@0.0.8 - -## 0.0.8 - -### Patch Changes - -- Updated dependencies - - @openfn/runtime@0.0.7 - -## 0.0.7 - -### Patch Changes - -- Updated dependencies - - @openfn/runtime@0.0.6 - -## 0.0.6 - -### Patch Changes - -- Updated dependencies [f79bf9a] - - @openfn/compiler@0.0.8 - - @openfn/runtime@0.0.5 - -## 0.0.5 - -### Patch Changes - -- Updated dependencies [5623913] - - @openfn/compiler@0.0.7 - - @openfn/runtime@0.0.4 - -## 0.0.4 - -### Patch Changes - -- @openfn/compiler@0.0.6 - -## 0.0.3 - -### Patch Changes - -- 8148cd5: Updated builds -- Updated dependencies [8148cd5] - - @openfn/compiler@0.0.5 - - @openfn/runtime@0.0.3 - -## 0.0.2 - -### Patch Changes - -- 3f6dc98: Initial release of new runtime, compiler and cli -- Updated dependencies [b5ce654] -- Updated dependencies [3f6dc98] - - @openfn/runtime@0.0.2 - - @openfn/compiler@0.0.4 +- Workerpool integration +- Isolated module loading with workflow id +- Timeout on the attempt +- Purging of threads while idle +- Autoinstall of modules diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 1d25db8a7..d59dc2213 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,10 +1,9 @@ { "name": "@openfn/engine-multi", - "version": "0.0.41", + "version": "0.1.0", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", - "private": true, "scripts": { "test": "pnpm ava", "test:types": "pnpm tsc --noEmit --project tsconfig.json", @@ -37,6 +36,7 @@ }, "files": [ "dist", - "README.md" + "README.md", + "CHANGELOG.md" ] } diff --git a/packages/engine-multi/test.js b/packages/engine-multi/test.js deleted file mode 100644 index 0ecb6918b..000000000 --- a/packages/engine-multi/test.js +++ /dev/null @@ -1,29 +0,0 @@ -import createAPI from './dist/index.js'; -import createLogger from '@openfn/logger'; - -const api = createAPI({ - logger: createLogger(null, { level: 'debug' }), - // Disable compilation - compile: { - skip: true, - }, -}); - -const plan = { - id: 'a', - jobs: [ - { - expression: `export default [s => s]`, - // with no adaptor it shouldn't try to autoinstall - }, - ], -}; - -// this basically works so long as --experimental-vm-modules is on -// although the event doesn't feed through somehow, but that's different -const listener = api.execute(plan); -listener.on('workflow-complete', ({ state }) => { - console.log(state); - console.log('workflow completed'); - process.exit(0); -}); From 719ff4b947a1a04475311bd322cd9a0467d83642 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 12:55:15 +0000 Subject: [PATCH 221/232] pnpm: exclude dummy repo from workspace --- pnpm-lock.yaml | 109 ++++++++++++++++++++++++++++++++++++++------ pnpm-workspace.yaml | 4 ++ 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b7ad0667a..87f069789 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -104,6 +104,12 @@ importers: specifier: ^3.0.2 version: 3.0.2 + integration-tests/cli/repo: + dependencies: + '@openfn/language-common_1.11.1': + specifier: npm:@openfn/language-common@^1.11.1 + version: /@openfn/language-common@1.11.1 + integration-tests/worker: dependencies: '@openfn/engine-multi': @@ -146,12 +152,6 @@ importers: specifier: ^5.1.6 version: 5.1.6 - integration-tests/worker/dummy-repo: - dependencies: - '@openfn/stateful-test_1.0.0': - specifier: '@npm:@openfn/stateful-test@1.0.0' - version: link:@npm:@openfn/stateful-test@1.0.0 - packages/cli: dependencies: '@inquirer/prompts': @@ -430,6 +430,10 @@ importers: specifier: ^5.1.6 version: 5.1.6 + packages/engine-multi/tmp/a/b/c: {} + + packages/engine-multi/tmp/repo: {} + packages/lightning-mock: dependencies: '@koa/router': @@ -1356,6 +1360,11 @@ packages: heap: 0.2.7 dev: false + /@fastify/busboy@2.0.0: + resolution: {integrity: sha512-JUFJad5lv7jxj926GPgymrWQxxjPYuJNiNjNMzqT+HiuP6Vl3dk5xzG+8sTX96np0ZAluvaMzPsjhHZ5rNuNQQ==} + engines: {node: '>=14'} + dev: false + /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1621,6 +1630,21 @@ packages: semver: 7.5.4 dev: true + /@openfn/language-common@1.11.1: + resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==} + dependencies: + ajv: 8.12.0 + axios: 1.1.3 + csv-parse: 5.5.2 + csvtojson: 2.0.10 + date-fns: 2.30.0 + jsonpath-plus: 4.0.0 + lodash: 4.17.21 + undici: 5.27.0 + transitivePeerDependencies: + - debug + dev: false + /@openfn/language-common@1.7.5: resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==} dependencies: @@ -2088,6 +2112,15 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 + /ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + dev: false + /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -2248,7 +2281,6 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -2396,7 +2428,6 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug - dev: true /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -2477,6 +2508,10 @@ packages: readable-stream: 4.2.0 dev: true + /bluebird@3.7.2: + resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} + dev: false + /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} @@ -2873,7 +2908,6 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 - dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -3016,6 +3050,10 @@ packages: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} dev: true + /csv-parse@5.5.2: + resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} + dev: false + /csv-stringify@5.6.5: resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==} dev: true @@ -3030,6 +3068,16 @@ packages: stream-transform: 2.1.3 dev: true + /csvtojson@2.0.10: + resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} + engines: {node: '>=4.0.0'} + hasBin: true + dependencies: + bluebird: 3.7.2 + lodash: 4.17.21 + strip-bom: 2.0.0 + dev: false + /currently-unhandled@0.4.1: resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==} engines: {node: '>=0.10.0'} @@ -3170,7 +3218,6 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} - dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3996,6 +4043,10 @@ packages: - supports-color dev: true + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: false + /fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} @@ -4152,7 +4203,6 @@ packages: peerDependenciesMeta: debug: optional: true - dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4183,7 +4233,6 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 - dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -4961,6 +5010,10 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} + /is-utf8@0.2.1: + resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} + dev: false + /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -5054,6 +5107,10 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + dev: false + /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5068,7 +5125,6 @@ packages: /jsonpath-plus@4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} - dev: true /jsonpath@1.1.1: resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} @@ -6393,7 +6449,6 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -6426,7 +6481,6 @@ packages: /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} - dev: true /qs@6.11.2: resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} @@ -6632,6 +6686,11 @@ packages: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true @@ -7167,6 +7226,13 @@ packages: dependencies: ansi-regex: 6.0.1 + /strip-bom@2.0.0: + resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} + engines: {node: '>=0.10.0'} + dependencies: + is-utf8: 0.2.1 + dev: false + /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -7720,6 +7786,13 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true + /undici@5.27.0: + resolution: {integrity: sha512-l3ydWhlhOJzMVOYkymLykcRRXqbUaQriERtR70B9LzNkZ4bX52Fc8wbTDneMiwo8T+AemZXvXaTx+9o5ROxrXg==} + engines: {node: '>=14.0'} + dependencies: + '@fastify/busboy': 2.0.0 + dev: false + /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'} @@ -7770,6 +7843,12 @@ packages: engines: {node: '>=4'} dev: true + /uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + dependencies: + punycode: 2.3.0 + dev: false + /urix@0.1.0: resolution: {integrity: sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==} deprecated: Please see https://github.com/lydell/urix#deprecated diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8d06e49d5..d980b74a6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,11 @@ packages: + # exclude the integration test repo + - '!integration-tests/worker/dummy-repo/**' + # all packages in subdirs of packages/ and components/ - 'packages/**' - 'examples/**' - 'integration-tests/**' + # exclude packages that are inside test directories - '!**/test/**' From dc4a1978ac54c49e190cb79ffac86e7037c70730 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 15:46:58 +0000 Subject: [PATCH 222/232] Engine: refactor timeout error --- packages/engine-multi/src/api/execute.ts | 14 ++++++-- packages/engine-multi/src/errors.ts | 36 +++++++++++++++++-- .../engine-multi/test/api/execute.test.ts | 2 +- packages/engine-multi/test/engine.test.ts | 2 +- 4 files changed, 46 insertions(+), 8 deletions(-) diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 0d1b7283c..6827466aa 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -1,4 +1,6 @@ // Execute a compiled workflow +import { Promise as WorkerPoolPromise } from 'workerpool'; + import * as workerEvents from '../worker/events'; import { ExecutionContext } from '../types'; @@ -13,6 +15,7 @@ import { error, } from './lifecycle'; import preloadCredentials from './preload-credentials'; +import { TimeoutError } from '../errors'; const execute = async (context: ExecutionContext) => { const { state, callWorker, logger, options } = context; @@ -57,9 +60,14 @@ const execute = async (context: ExecutionContext) => { events, options.timeout ).catch((e: any) => { - // E could be: - // A timeout - // An crash error within the job + // An error here is basically a crash state + + if (e instanceof WorkerPoolPromise.TimeoutError) { + // Map the workerpool error to our own + e = new TimeoutError(options.timeout!); + } + + // TODO: map anything else to an executionError // TODO what information can I usefully provide here? // DO I know which job I'm on? diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts index 708bbf736..877e65956 100644 --- a/packages/engine-multi/src/errors.ts +++ b/packages/engine-multi/src/errors.ts @@ -1,5 +1,35 @@ // This will eventually contain all the error classes thrown by the engine -import { Promise as WorkerPoolPromise } from 'workerpool'; -// Thrown when the whole workflow is timedout -export const TimeoutError = WorkerPoolPromise.TimeoutError; +export class EngineError extends Error { + source = 'worker'; + + severity = '-'; // subclasses MUST provide this! +} + +// This is thrown if a workflow takes too long to run +// It is generated by workerpool and thrown if the workerpool promise fails to resolve +export class TimeoutError extends EngineError { + severity = 'crash'; + type = 'TimeoutError'; + duration; + constructor(durationInMs: number) { + super(); + this.duration = durationInMs; + + if (durationInMs) { + this.message = `Workflow failed to return within ${durationInMs}ms`; + } else { + this.message = `Workflow failed to return within the specified time limit`; + } + } +} + +// This is a catch-all error thrown during execution +export class ExecutionError extends EngineError { + severity = 'crash'; + original: any; // this is the original error + constructor(original: any) { + super(); + this.original = original; + } +} diff --git a/packages/engine-multi/test/api/execute.test.ts b/packages/engine-multi/test/api/execute.test.ts index 4bad58dee..ce574da1a 100644 --- a/packages/engine-multi/test/api/execute.test.ts +++ b/packages/engine-multi/test/api/execute.test.ts @@ -182,7 +182,7 @@ test.serial('should emit error on timeout', async (t) => { t.truthy(event.threadId); t.is(event.type, 'TimeoutError'); - t.assert(event.message.match(/Promise timed out after/)); + t.regex(event.message, /failed to return within 10ms/); }); // how will we test compilation? diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index b0592d65a..23a34357c 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -227,7 +227,7 @@ test('timeout the whole attempt and emit an error', async (t) => { engine.listen(plan.id, { [e.WORKFLOW_ERROR]: ({ message, type }) => { t.is(type, 'TimeoutError'); - t.is(message, 'Promise timed out after 10 ms'); + t.regex(message, /failed to return within 10ms/); done(); }, }); From ee89abbda9a1dc92d3b6d2b10fcfd67ebf19e917 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 17:37:36 +0000 Subject: [PATCH 223/232] worker: be more robust when connecting to lightning --- packages/ws-worker/package.json | 1 - packages/ws-worker/src/server.ts | 177 +++++++++++++++++++------------ packages/ws-worker/src/start.ts | 4 + 3 files changed, 111 insertions(+), 71 deletions(-) diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index a7959efba..2de87fc1a 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -10,7 +10,6 @@ "build": "tsup --config ../../tsup.config.js src/index.ts src/mock-worker.ts --no-splitting", "build:watch": "pnpm build --watch", "start": "NODE_OPTIONS=\"--experimental-vm-modules --no-warnings\" ts-node-esm --transpile-only src/start.ts", - "start:lightning": "ts-node-esm --transpile-only src/mock/lightning/start.ts", "start:watch": "nodemon -e ts,js --watch ../runtime-manager/dist --watch ./src --exec 'pnpm start'", "pack": "pnpm pack --pack-destination ../../dist" }, diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index afb3d53bd..79a46f05b 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -3,6 +3,7 @@ import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import Router from '@koa/router'; import { createMockLogger, Logger } from '@openfn/logger'; +import { RuntimeEngine } from '@openfn/engine-multi'; import startWorkloop from './api/workloop'; import claim from './api/claim'; @@ -23,13 +24,78 @@ type ServerOptions = { secret?: string; // worker secret }; -function createServer(engine: any, options: ServerOptions = {}) { +// this is the server/koa API +interface ServerApp extends Koa { + socket: any; + channel: any; + + execute: ({ id, token }: CLAIM_ATTEMPT) => Promise; + destroy: () => void; + killWorkloop: () => void; +} + +const DEFAULT_PORT = 1234; + +// TODO move out into another file, make testable +function connect( + app: ServerApp, + engine: RuntimeEngine, + logger: Logger, + options: ServerOptions = {} +) { + logger.debug('Connecting to Lightning at', options.lightning); + + connectToLightning(options.lightning!, engine.id, options.secret!) + .then(({ socket, channel }) => { + logger.success('Connected to Lightning at', options.lightning); + + // save the channel and socket + app.socket = socket; + app.channel = channel; + + // trigger the workloop + if (!options.noLoop) { + logger.info('Starting workloop'); + // TODO maybe namespace the workloop logger differently? It's a bit annoying + app.killWorkloop = startWorkloop(channel, app.execute, logger, { + maxBackoff: options.maxBackoff, + // timeout: 1000 * 60, // TMP debug poll once per minute + }); + } else { + logger.break(); + logger.warn('Workloop not starting'); + logger.info('This server will not auto-pull work from lightning.'); + logger.info('You can manually claim by posting to /claim, eg:'); + logger.info( + ` curl -X POST http://locahost:${options.port || DEFAULT_PORT}/claim` + ); + logger.break(); + } + }) + .catch((e) => { + logger.error( + 'CRITICAL ERROR: could not connect to lightning at', + options.lightning + ); + logger.debug(e); + + app.killWorkloop?.(); + + // Try to Reconnect after 10 seconds + setTimeout(() => { + connect(app, engine, logger, options); + }, 1e4); + }); +} + +function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { const logger = options.logger || createMockLogger(); - const port = options.port || 1234; + const port = options.port || DEFAULT_PORT; logger.debug('Starting server'); - const app = new Koa(); + const app = new Koa() as ServerApp; + const router = new Router(); app.use(bodyParser()); app.use( @@ -41,84 +107,55 @@ function createServer(engine: any, options: ServerOptions = {}) { const server = app.listen(port); logger.success('ws-worker listening on', port); - let killWorkloop: () => void; + // TODO this probably needs to move into ./api/ somewhere + app.execute = async ({ id, token }: CLAIM_ATTEMPT) => { + if (app.socket) { + // TODO need to verify the token against LIGHTNING_PUBLIC_KEY + const { + channel: attemptChannel, + plan, + options, + } = await joinAttemptChannel(app.socket, token, id, logger); + execute(attemptChannel, engine, logger, plan, options); + } else { + logger.error('No lightning socket established'); + // TODO something else. Throw? Emit? + } + }; - (app as any).destroy = () => { - // TODO close the work loop - logger.info('Closing server'); + // Debug API to manually trigger a claim + router.post('/claim', async (ctx) => { + logger.info('triggering claim from POST request'); + return claim(app.channel, app.execute, logger) + .then(() => { + logger.info('claim complete: 1 attempt claimed'); + ctx.body = 'complete'; + ctx.status = 200; + }) + .catch(() => { + logger.info('claim complete: no attempts'); + ctx.body = 'no attempts'; + ctx.status = 204; + }); + }); + + app.destroy = () => { + logger.info('Closing server...'); server.close(); - killWorkloop?.(); + app.killWorkloop?.(); + logger.success('Server closed'); }; - const router = new Router(); app.use(router.routes()); if (options.lightning) { - logger.debug('Connecting to Lightning at', options.lightning); - // TODO this is too hard to unit test, need to pull it out - connectToLightning(options.lightning, engine.id, options.secret!) - .then(({ socket, channel }) => { - logger.success('Connected to Lightning at', options.lightning); - - const startAttempt = async ({ id, token }: CLAIM_ATTEMPT) => { - // TODO need to verify the token against LIGHTNING_PUBLIC_KEY - const { - channel: attemptChannel, - plan, - options, - } = await joinAttemptChannel(socket, token, id, logger); - execute(attemptChannel, engine, logger, plan, options); - }; - - if (!options.noLoop) { - logger.info('Starting workloop'); - // TODO maybe namespace the workloop logger differently? It's a bit annoying - killWorkloop = startWorkloop(channel, startAttempt, logger, { - maxBackoff: options.maxBackoff, - // timeout: 1000 * 60, // TMP debug poll once per minute - }); - } else { - logger.break(); - logger.warn('Workloop not starting'); - logger.info('This server will not auto-pull work from lightning.'); - logger.info('You can manually claim by posting to /claim, eg:'); - logger.info(` curl -X POST http://locahost:${port}/claim`); - logger.break(); - } - - // debug/unit test API to run a workflow - // TODO Only loads in dev mode? - (app as any).execute = startAttempt; - - // Debug API to manually trigger a claim - router.post('/claim', async (ctx) => { - logger.info('triggering claim from POST request'); - return claim(channel, startAttempt, logger) - .then(() => { - logger.info('claim complete: 1 attempt claimed'); - ctx.body = 'complete'; - ctx.status = 200; - }) - .catch(() => { - logger.info('claim complete: no attempts'); - ctx.body = 'no attempts'; - ctx.status = 204; - }); - }); - }) - .catch((e) => { - logger.error( - 'CRITICAL ERROR: could not connect to lightning at', - options.lightning - ); - logger.debug(e); - process.exit(1); - }); + connect(app, engine, logger, options); } else { logger.warn('No lightning URL provided'); } - // TMP doing this for tests but maybe its better done externally + // TMP doing this for tests but maybe its better done externally? + // @ts-ignore app.on = (...args) => { return engine.on(...args); }; diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts index 0813315ab..24ff1cbf2 100644 --- a/packages/ws-worker/src/start.ts +++ b/packages/ws-worker/src/start.ts @@ -63,6 +63,10 @@ const logger = createLogger('SRV', { level: args.log }); if (args.lightning === 'mock') { args.lightning = 'ws://localhost:8888/worker'; + if (!args.secret) { + // Set a fake secret to stop the console warning + args.secret = 'abdefg'; + } } else if (!args.secret) { const { WORKER_SECRET } = process.env; if (!WORKER_SECRET) { From 4874e87cccb3b50f084b64e9c6dd6692ef0537fe Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 17:43:17 +0000 Subject: [PATCH 224/232] worker: more connection robustness --- packages/ws-worker/src/api/connect.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/src/api/connect.ts b/packages/ws-worker/src/api/connect.ts index 5028659b4..3d6bfdcce 100644 --- a/packages/ws-worker/src/api/connect.ts +++ b/packages/ws-worker/src/api/connect.ts @@ -25,11 +25,15 @@ export const connectToLightning = ( transport: WebSocket, }); + let didOpen = false; + // TODO need error & timeout handling (ie wrong endpoint or endpoint offline) // Do we infinitely try to reconnect? // Consider what happens when the connection drops // Unit tests on all of these behaviours! socket.onOpen(() => { + didOpen = true; + // join the queue channel // TODO should this send the worker token? const channel = socket.channel('worker:queue') as Channel; @@ -47,9 +51,15 @@ export const connectToLightning = ( }); }); - // TODO what even happens if the connection fails? + // if we fail to connect socket.onError((e: any) => { - reject(e); + // If we failed to connect, reject the promise + // The server will try and reconnect itself.s + if (!didOpen) { + reject(e); + } + // Note that if we DID manage to connect once, the socket should re-negotiate + // wihout us having to do anything }); socket.connect(); From 60ed0f2a00463ac83d77b05d634b959e72f0fb78 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 17:46:49 +0000 Subject: [PATCH 225/232] worker: refactor connect -> channels/worker-queue --- packages/ws-worker/src/api/channels/workers | 3 --- .../ws-worker/src/{api => }/channels/attempts | 0 .../connect.ts => channels/worker-queue.ts} | 5 +++-- packages/ws-worker/src/server.ts | 4 ++-- .../worker-queue.test.ts} | 20 +++++++++++++------ packages/ws-worker/tsconfig.json | 2 +- 6 files changed, 20 insertions(+), 14 deletions(-) delete mode 100644 packages/ws-worker/src/api/channels/workers rename packages/ws-worker/src/{api => }/channels/attempts (100%) rename packages/ws-worker/src/{api/connect.ts => channels/worker-queue.ts} (96%) rename packages/ws-worker/test/{api/connect.test.ts => channels/worker-queue.test.ts} (79%) diff --git a/packages/ws-worker/src/api/channels/workers b/packages/ws-worker/src/api/channels/workers deleted file mode 100644 index 17d003acd..000000000 --- a/packages/ws-worker/src/api/channels/workers +++ /dev/null @@ -1,3 +0,0 @@ -// todo all the worker channel stuff goes here -// connect to worker -// handle auth diff --git a/packages/ws-worker/src/api/channels/attempts b/packages/ws-worker/src/channels/attempts similarity index 100% rename from packages/ws-worker/src/api/channels/attempts rename to packages/ws-worker/src/channels/attempts diff --git a/packages/ws-worker/src/api/connect.ts b/packages/ws-worker/src/channels/worker-queue.ts similarity index 96% rename from packages/ws-worker/src/api/connect.ts rename to packages/ws-worker/src/channels/worker-queue.ts index 3d6bfdcce..d44c848c0 100644 --- a/packages/ws-worker/src/api/connect.ts +++ b/packages/ws-worker/src/channels/worker-queue.ts @@ -1,5 +1,6 @@ import { Socket as PhxSocket } from 'phoenix'; import { WebSocket } from 'ws'; + import generateWorkerToken from '../util/worker-token'; import type { Socket, Channel } from '../types'; @@ -8,7 +9,7 @@ type SocketAndChannel = { channel: Channel; }; -export const connectToLightning = ( +const connectToWorkerQueue = ( endpoint: string, serverId: string, secret: string, @@ -66,4 +67,4 @@ export const connectToLightning = ( }); }; -export default connectToLightning; +export default connectToWorkerQueue; diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 79a46f05b..64b74bf79 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -9,7 +9,7 @@ import startWorkloop from './api/workloop'; import claim from './api/claim'; import { execute } from './api/execute'; import joinAttemptChannel from './api/start-attempt'; -import connectToLightning from './api/connect'; +import connectToWorkerQueue from './channels/worker-queue'; import { CLAIM_ATTEMPT } from './events'; type ServerOptions = { @@ -45,7 +45,7 @@ function connect( ) { logger.debug('Connecting to Lightning at', options.lightning); - connectToLightning(options.lightning!, engine.id, options.secret!) + connectToWorkerQueue(options.lightning!, engine.id, options.secret!) .then(({ socket, channel }) => { logger.success('Connected to Lightning at', options.lightning); diff --git a/packages/ws-worker/test/api/connect.test.ts b/packages/ws-worker/test/channels/worker-queue.test.ts similarity index 79% rename from packages/ws-worker/test/api/connect.test.ts rename to packages/ws-worker/test/channels/worker-queue.test.ts index 3c11957eb..f82e65b8b 100644 --- a/packages/ws-worker/test/api/connect.test.ts +++ b/packages/ws-worker/test/channels/worker-queue.test.ts @@ -1,10 +1,15 @@ import test from 'ava'; import * as jose from 'jose'; -import connect from '../../src/api/connect'; +import connectToWorkerQueue from '../../src/channels/worker-queue'; import { mockSocket } from '../../src/mock/sockets'; test('should connect', async (t) => { - const { socket, channel } = await connect('www', 'a', 'secret', mockSocket); + const { socket, channel } = await connectToWorkerQueue( + 'www', + 'a', + 'secret', + mockSocket + ); t.truthy(socket); t.truthy(socket.connect); @@ -27,7 +32,7 @@ test('should connect with an auth token', async (t) => { return socket; } - const { socket, channel } = await connect( + const { socket, channel } = await connectToWorkerQueue( 'www', workerId, secret, @@ -59,9 +64,12 @@ test('should fail to connect with an invalid auth token', async (t) => { return socket; } - await t.throwsAsync(connect('www', workerId, 'wrong-secret!', createSocket), { - message: 'auth_fail', - }); + await t.throwsAsync( + connectToWorkerQueue('www', workerId, 'wrong-secret!', createSocket), + { + message: 'auth_fail', + } + ); }); // TODO maybe? diff --git a/packages/ws-worker/tsconfig.json b/packages/ws-worker/tsconfig.json index 7a3e3bb21..05365c374 100644 --- a/packages/ws-worker/tsconfig.json +++ b/packages/ws-worker/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.common", - "include": ["src/**/*.ts", "test/mock/data.ts", "src/api/channels/attempts"], + "include": ["src/**/*.ts", "test/mock/data.ts", "src/channels/attempts"], "compilerOptions": { "module": "ESNext" } From 8c83d54fe0867a147691bf9411b160d28aee16bd Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 17:49:23 +0000 Subject: [PATCH 226/232] worker: refactor start-attempt -> channels/attempt --- .../src/{api/start-attempt.ts => channels/attempt.ts} | 0 packages/ws-worker/src/channels/attempts | 7 ------- packages/ws-worker/src/server.ts | 2 +- .../start-attempt.test.ts => channels/attempt.test.ts} | 3 +-- 4 files changed, 2 insertions(+), 10 deletions(-) rename packages/ws-worker/src/{api/start-attempt.ts => channels/attempt.ts} (100%) delete mode 100644 packages/ws-worker/src/channels/attempts rename packages/ws-worker/test/{api/start-attempt.test.ts => channels/attempt.test.ts} (95%) diff --git a/packages/ws-worker/src/api/start-attempt.ts b/packages/ws-worker/src/channels/attempt.ts similarity index 100% rename from packages/ws-worker/src/api/start-attempt.ts rename to packages/ws-worker/src/channels/attempt.ts diff --git a/packages/ws-worker/src/channels/attempts b/packages/ws-worker/src/channels/attempts deleted file mode 100644 index 39c089914..000000000 --- a/packages/ws-worker/src/channels/attempts +++ /dev/null @@ -1,7 +0,0 @@ -// @ts-ignore - -// all the attempt stuff goes here -// join the attempt channel -// do we pull the attempt data here? I don't think so really tbh -// it shoud prob be in execute, which is kinda annoying ebcause executeis designed to take a plan -// i guess it just takes a cahannel and it pulls diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 64b74bf79..8f9fe786c 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -8,7 +8,7 @@ import { RuntimeEngine } from '@openfn/engine-multi'; import startWorkloop from './api/workloop'; import claim from './api/claim'; import { execute } from './api/execute'; -import joinAttemptChannel from './api/start-attempt'; +import joinAttemptChannel from './channels/attempt'; import connectToWorkerQueue from './channels/worker-queue'; import { CLAIM_ATTEMPT } from './events'; diff --git a/packages/ws-worker/test/api/start-attempt.test.ts b/packages/ws-worker/test/channels/attempt.test.ts similarity index 95% rename from packages/ws-worker/test/api/start-attempt.test.ts rename to packages/ws-worker/test/channels/attempt.test.ts index 064067469..e8d37d6a3 100644 --- a/packages/ws-worker/test/api/start-attempt.test.ts +++ b/packages/ws-worker/test/channels/attempt.test.ts @@ -1,8 +1,7 @@ import test from 'ava'; import { mockSocket, mockChannel } from '../../src/mock/sockets'; -import joinAttemptChannel from '../../src/api/start-attempt'; +import joinAttemptChannel, { loadAttempt } from '../../src/channels/attempt'; import { GET_ATTEMPT } from '../../src/events'; -import { loadAttempt } from '../../src/api/start-attempt'; import { attempts } from '../mock/data'; import { createMockLogger } from '@openfn/logger'; From d5fd227fd6d9d6b3350abdade106aada22eb0f95 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 17:55:30 +0000 Subject: [PATCH 227/232] worker: add human id --- packages/ws-worker/package.json | 3 ++- packages/ws-worker/src/server.ts | 5 ++++- pnpm-lock.yaml | 8 ++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 2de87fc1a..822810962 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -23,6 +23,7 @@ "@types/koa-logger": "^3.1.2", "@types/ws": "^8.5.6", "fast-safe-stringify": "^2.1.1", + "human-id": "^4.1.0", "jose": "^4.14.6", "koa": "^2.13.4", "koa-bodyparser": "^4.4.0", @@ -31,6 +32,7 @@ "ws": "^8.14.1" }, "devDependencies": { + "@openfn/lightning-mock": "workspace:*", "@types/koa": "^2.13.5", "@types/koa-bodyparser": "^4.3.10", "@types/koa-route": "^3.2.6", @@ -40,7 +42,6 @@ "@types/nodemon": "1.19.3", "@types/phoenix": "^1.6.2", "@types/yargs": "^17.0.12", - "@openfn/lightning-mock": "workspace:*", "ava": "5.1.0", "koa-route": "^3.2.0", "koa-websocket": "^7.0.0", diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 8f9fe786c..26a4cde53 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -2,6 +2,7 @@ import Koa from 'koa'; import bodyParser from 'koa-bodyparser'; import koaLogger from 'koa-logger'; import Router from '@koa/router'; +import { humanId } from 'human-id'; import { createMockLogger, Logger } from '@openfn/logger'; import { RuntimeEngine } from '@openfn/engine-multi'; @@ -26,6 +27,7 @@ type ServerOptions = { // this is the server/koa API interface ServerApp extends Koa { + id: string; socket: any; channel: any; @@ -95,6 +97,7 @@ function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { logger.debug('Starting server'); const app = new Koa() as ServerApp; + app.id = humanId({ separator: '-', capitalize: false }); const router = new Router(); app.use(bodyParser()); @@ -105,7 +108,7 @@ function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { ); const server = app.listen(port); - logger.success('ws-worker listening on', port); + logger.success(`ws-worker ${app.id} listening on ${port}`); // TODO this probably needs to move into ./api/ somewhere app.execute = async ({ id, token }: CLAIM_ATTEMPT) => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87f069789..c174da7b7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -631,6 +631,9 @@ importers: fast-safe-stringify: specifier: ^2.1.1 version: 2.1.1 + human-id: + specifier: ^4.1.0 + version: 4.1.0 jose: specifier: ^4.14.6 version: 4.14.6 @@ -4622,6 +4625,11 @@ packages: resolution: {integrity: sha512-UNopramDEhHJD+VR+ehk8rOslwSfByxPIZyJRfV739NDhN5LF1fa1MqnzKm2lGTQRjNrjK19Q5fhkgIfjlVUKw==} dev: true + /human-id@4.1.0: + resolution: {integrity: sha512-WYeWmHXGo1ZPGZuy7I2c+sgM83GvlQR1jrF1zLX6fID9JEVGkZgZe8PHj5HacWC7d9V0rNc4iRVhb9QmO55CUQ==} + hasBin: true + dev: false + /human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} From e55074ea9684e4179025a4d05cdccfa03c5b48b5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 18:01:20 +0000 Subject: [PATCH 228/232] worker: fix sever id --- packages/ws-worker/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 26a4cde53..0c37f6e94 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -47,7 +47,7 @@ function connect( ) { logger.debug('Connecting to Lightning at', options.lightning); - connectToWorkerQueue(options.lightning!, engine.id, options.secret!) + connectToWorkerQueue(options.lightning!, app.id, options.secret!) .then(({ socket, channel }) => { logger.success('Connected to Lightning at', options.lightning); From 0643ed61cb530b7bce5ab3b772180057f656fc7e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 31 Oct 2023 18:07:36 +0000 Subject: [PATCH 229/232] comment --- packages/ws-worker/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 0c37f6e94..b28235c61 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -38,7 +38,7 @@ interface ServerApp extends Koa { const DEFAULT_PORT = 1234; -// TODO move out into another file, make testable +// TODO move out into another file, make testable, test in isolation function connect( app: ServerApp, engine: RuntimeEngine, From 07da5857421096c7c369336c59a1d0bc3290b2df Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 1 Nov 2023 12:15:37 +0000 Subject: [PATCH 230/232] Docs --- packages/lightning-mock/README.md | 49 ++++++++++++++++++++- packages/ws-worker/CHANGELOG.md | 12 ++++++ packages/ws-worker/README.md | 71 +++++++++++-------------------- 3 files changed, 85 insertions(+), 47 deletions(-) create mode 100644 packages/ws-worker/CHANGELOG.md diff --git a/packages/lightning-mock/README.md b/packages/lightning-mock/README.md index 3bf01c1c1..87272c6ff 100644 --- a/packages/lightning-mock/README.md +++ b/packages/lightning-mock/README.md @@ -1,10 +1,55 @@ ## Lightning Mock -This package contains a mock lightning server, designed to be used with the worker +This package contains a mock lightning server, designed to be used with the worker and test suites. -It is mostly used for unit and integration tests, but it can be run standalone from the command line: +It is currently private and only used by this monorepo. + +## Getting Started ``` pnpm install pnpm start ``` + +This will run on port 8888. Add `-p` to change the port. + +## Architecture + +This repo contains: + +- A mock Pheonix websoket server implementation. Barebones but compatible with the phoenix sockets client +- A mock Lightning server which handles and acknowledges Attempt comms and an Attempts queue + +The key API is in `src/api-socket/ts`. The `createSocketAPI` function hooks up websockets and binds events to event handlers. It's supposed to be quite declarative so you can track the API quite easily. + +See `src/events.ts` for a typings of the expected event names, payloads and replies. + +Additional dev-time API's can be found in `src/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. + +## Usage + +The server exposes a small dev API allowing you to post an Attempt. + +You can add an attempt (`{ jobs, triggers, edges }`) to the queue with: + +``` +curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" +``` + +Here's an example attempt: + +``` +{ + "id": "my-attempt, + "triggers": [], + "edges": [], + "jobs": [ + { + "id": "job1", + "state": { "data": { "done": true } }, + "adaptor": "@openfn/language-common@1.7.7", + "body": "{ \"result\": 42 }" + } + ] +} +``` diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md new file mode 100644 index 000000000..20fbca0dd --- /dev/null +++ b/packages/ws-worker/CHANGELOG.md @@ -0,0 +1,12 @@ +# ws-worker + +## 0.1.0 + +First release of the websocket worker, which handles comm between Lightning and the multi-threaded engine. + +Features: + +- Websocket integration with JWT auth +- Eventing between Lightning and the Worker +- Eventing between the Worker and the Engine +- Placeholder exit reasons diff --git a/packages/ws-worker/README.md b/packages/ws-worker/README.md index 87f7246cb..d8f5f55b4 100644 --- a/packages/ws-worker/README.md +++ b/packages/ws-worker/README.md @@ -2,75 +2,66 @@ The Websocket Worker `ws-worker` provides a Websocket interface between Lightning and a Runtime Engine. -It is a fairly thin layer between the two systems, designed to transport messages and convert Lightning data structres into runtime-friendly ones. +It is a fairly thin layer between the two systems, designed to transport messages and convert Lightning data structures into runtime-friendly ones. This package contains: -- A mock Lightning implementation -- A mock runtime engine implementation -- A mock server for phoenix websockets (allowing the phx Socket client to connect and exchange messages) - A server which connects Lightning to an Engine (exposing dev APIs to http and node.js) +- A mock runtime engine implementation The mock services allow lightweight and controlled testing of the interfaces between them. ## Getting started -There are several components you may want to run to get started. - -If you're running a local lightning server, remember to set WORKER_SECRET. +To use this server: -### WS Server +- Start a lightning instance (you can use the mock if you like, see `../lightning-mock`) +- Start the worker server with `pnpm start` -To start a `ws-socket` server, run: +The worker will use the WORKER_SECRET env var (which you should have set for Lightning already). Check WORKERS.md in Lightning and run this in Lightning if you haven't already: ``` -pnpm start +mix lightning.gen_worker_keys ``` -This will try and connect to lightning at `localhost:4000`. Use `-l mock` to connect to the default mock server from this repo (you'll need to start the lightning server), or pass your own url to a lightning isntance. +### WS Server -You can start a dev server (which rebuilds on save) by running: +To start a `ws-worker` server, run: ``` -pnpm start:watch +pnpm start ``` -This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`. +You may want to add `--log debug` or disable the work loop, see below. + +The default settings will try and connect to lightning at `localhost:4000`. -To connect to a lightning instance, pass the `-l` flag. +Pass a custom lightining url with `-l ws://localhost:1234`. You need to include the websocket endpoint, which at the time of writing is `/worker`. -### Lightning Mock +Use `-l mock` to connect to a lightning mock server (on the default port). -You can start a Lightning mock server with: +## Watched Server + +You can start a dev server (which rebuilds on save) by running: ``` -pnpm start:lightning +pnpm start:watch ``` -This will run on port 8888 [TODO: drop yargs in to customise the port] +This will wrap a real runtime engine into the server (?). It will rebuild when the Worker Engine code changes (although you'll have to `pnpm build:watch` in `runtime-manager`). This will use the repo at `ENGINE_REPO_DIR` or `/tmp/openfn/repo`. + +### Disabling auto-fetch -You can add an attempt (`{ jobs, triggers, edges }`) to the queue with: +When working in dev it is convinient to disable the workloop. To switch it off, run: ``` -curl -X POST http://localhost:8888/attempt -d @tmp/my-attempt.json -H "Content-Type: application/json" +pnpm start --no-loop ``` -Here's an example attempt: +To manually trigger a claim, post to `/claim`: ``` -{ - "id": "my-attempt, - "triggers": [], - "edges": [], - "jobs": [ - { - "id": "job1", - "state": { "data": { "done": true } }, - "adaptor": "@openfn/language-common@1.7.7", - "body": "{ \"result\": 42 }" - } - ] -} +curl -X POST http://localhost:2222/claim ``` ## Architecture @@ -80,13 +71,3 @@ Lightning is expected to maintain a queue of attempts. The Worker pulls those at While the engine executes it may need to request more information (like credentials and dataclips) and may feedback status (such as logging and runs). The Worker satisifies both these requirements. The ws-worker server is designed for zero persistence. It does not have any database, does not use the file system. Should the server crash, tracking of any active jobs will be lost (Lightning is expected to time these runs out). - -### ws-worker - -### Lightning Mock - -The key API is in `src/mock/lightning/api-socket/ts`. The `createSocketAPI` function hooks up websockets and binds events to event handlers. It's supposed to be quite declarative so you can track the API quite easily. - -See `src/events.ts` for a typings of the expected event names, payloads and replies. - -Additional dev-time API's can be found in `src/mock/lightning/api-dev.ts`. These are for testing purposes only and not expected to be part of the Lightning platform. From 92e8a080255296a8ace47c61cc761b57eda5e350 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 1 Nov 2023 12:31:07 +0000 Subject: [PATCH 231/232] Bump versions, adjust dependencies --- packages/cli/CHANGELOG.md | 10 ++++++++++ packages/cli/package.json | 2 +- packages/compiler/CHANGELOG.md | 7 +++++++ packages/compiler/package.json | 2 +- packages/deploy/CHANGELOG.md | 7 +++++++ packages/deploy/package.json | 2 +- packages/engine-multi/package.json | 4 ---- packages/lightning-mock/CHANGELOG.md | 13 +++++++++++++ packages/lightning-mock/package.json | 2 +- packages/logger/CHANGELOG.md | 7 +++++++ packages/logger/package.json | 2 +- packages/runtime/CHANGELOG.md | 13 +++++++++++++ packages/runtime/package.json | 2 +- packages/ws-worker/package.json | 5 ----- 14 files changed, 63 insertions(+), 15 deletions(-) create mode 100644 packages/lightning-mock/CHANGELOG.md diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 5ed7d0956..4ac316689 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,15 @@ # @openfn/cli +## 0.4.2 + +### Patch Changes + +- Updated dependencies [1b6fa8e] + - @openfn/logger@0.0.18 + - @openfn/runtime@0.0.32 + - @openfn/compiler@0.0.37 + - @openfn/deploy@0.2.8 + ## 0.4.1 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index 690424a6a..35f1d5dcf 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "0.4.1", + "version": "0.4.2", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/compiler/CHANGELOG.md b/packages/compiler/CHANGELOG.md index 4abe4c174..e79ed3ea2 100644 --- a/packages/compiler/CHANGELOG.md +++ b/packages/compiler/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/compiler +## 0.0.37 + +### Patch Changes + +- Updated dependencies [1b6fa8e] + - @openfn/logger@0.0.18 + ## 0.0.36 ### Patch Changes diff --git a/packages/compiler/package.json b/packages/compiler/package.json index 80799dfe5..6b80680df 100644 --- a/packages/compiler/package.json +++ b/packages/compiler/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/compiler", - "version": "0.0.36", + "version": "0.0.37", "description": "Compiler and language tooling for openfn jobs.", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index d1f7e64cf..8171fc79e 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/deploy +## 0.2.8 + +### Patch Changes + +- Updated dependencies [1b6fa8e] + - @openfn/logger@0.0.18 + ## 0.2.7 ### Patch Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index cb4bbcc84..527c90d9e 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.2.7", + "version": "0.2.8", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index d59dc2213..2268547d9 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -18,16 +18,12 @@ "@openfn/language-common": "2.0.0-rc3", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", - "koa": "^2.13.4", "workerpool": "^6.5.1" }, "devDependencies": { - "@types/koa": "^2.13.5", "@types/node": "^18.15.13", - "@types/nodemon": "^1.19.2", "@types/workerpool": "^6.4.4", "ava": "5.3.1", - "nodemon": "^2.0.19", "ts-node": "^10.9.1", "tslib": "^2.4.0", "tsm": "^2.2.2", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md new file mode 100644 index 000000000..c7d9bbf5c --- /dev/null +++ b/packages/lightning-mock/CHANGELOG.md @@ -0,0 +1,13 @@ +# @openfn/lightning-mock + +## 1.0.1 + +### Patch Changes + +- Updated dependencies [7e4529e] +- Updated dependencies [d2360d4] +- Updated dependencies [195f098] +- Updated dependencies [1b6fa8e] + - @openfn/logger@0.0.18 + - @openfn/runtime@0.0.32 + - @openfn/engine-multi@0.1.1 diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index c32733291..7c479026b 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.0.0", + "version": "1.0.1", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/logger/CHANGELOG.md b/packages/logger/CHANGELOG.md index 62b68a685..623e51344 100644 --- a/packages/logger/CHANGELOG.md +++ b/packages/logger/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/logger +## 0.0.18 + +### Patch Changes + +- 7e4529e: Export SanitizePolicies type +- 1b6fa8e: Add proxy function + ## 0.0.17 ### Patch Changes diff --git a/packages/logger/package.json b/packages/logger/package.json index f73def522..5e3dd3ce0 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/logger", - "version": "0.0.17", + "version": "0.0.18", "description": "Cross-package logging utility", "module": "dist/index.js", "author": "Open Function Group ", diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index 3fbae2b91..ccd122e5e 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,18 @@ # @openfn/runtime +## 0.0.32 + +### Patch Changes + +- Add support to lazy load intial state and config +- d2360d4: Support a cacheKey to bust cached modules in long-running processes +- Add notify api +- 195f098: Trigger callbacks on job start, complete and init +- add deleteConfiguration option +- Fix intial state handling +- Updated dependencies [7e4529e] + - @openfn/logger@0.0.18 + ## 0.0.31 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index 233124b64..7b7228e74 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "0.0.31", + "version": "0.0.32", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 822810962..a4b359375 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -35,18 +35,13 @@ "@openfn/lightning-mock": "workspace:*", "@types/koa": "^2.13.5", "@types/koa-bodyparser": "^4.3.10", - "@types/koa-route": "^3.2.6", - "@types/koa-websocket": "^5.0.8", "@types/koa__router": "^12.0.1", "@types/node": "^18.15.3", "@types/nodemon": "1.19.3", "@types/phoenix": "^1.6.2", "@types/yargs": "^17.0.12", "ava": "5.1.0", - "koa-route": "^3.2.0", - "koa-websocket": "^7.0.0", "nodemon": "3.0.1", - "query-string": "^8.1.0", "ts-node": "^10.9.1", "tslib": "^2.4.0", "tsup": "^6.2.3", From 531c3e2a7ca3d15e1bde1021dfe8c45ec8725511 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 1 Nov 2023 12:31:24 +0000 Subject: [PATCH 232/232] Tidy --- .changeset/cyan-worms-clap.md | 5 --- .changeset/selfish-nails-live.md | 5 --- .changeset/shaggy-jars-brake.md | 5 --- .changeset/short-pens-punch.md | 5 --- .changeset/tough-coats-unite.md | 5 --- .changeset/warm-tables-explode.md | 5 --- integration-tests/worker/CHANGELOG.md | 12 +++++ integration-tests/worker/package.json | 2 +- pnpm-lock.yaml | 63 --------------------------- 9 files changed, 13 insertions(+), 94 deletions(-) delete mode 100644 .changeset/cyan-worms-clap.md delete mode 100644 .changeset/selfish-nails-live.md delete mode 100644 .changeset/shaggy-jars-brake.md delete mode 100644 .changeset/short-pens-punch.md delete mode 100644 .changeset/tough-coats-unite.md delete mode 100644 .changeset/warm-tables-explode.md create mode 100644 integration-tests/worker/CHANGELOG.md diff --git a/.changeset/cyan-worms-clap.md b/.changeset/cyan-worms-clap.md deleted file mode 100644 index e0b011c87..000000000 --- a/.changeset/cyan-worms-clap.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/logger': patch ---- - -Export SanitizePolicies type diff --git a/.changeset/selfish-nails-live.md b/.changeset/selfish-nails-live.md deleted file mode 100644 index a0b19b094..000000000 --- a/.changeset/selfish-nails-live.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/rtm-server': minor ---- - -First pass of runtime manager server" diff --git a/.changeset/shaggy-jars-brake.md b/.changeset/shaggy-jars-brake.md deleted file mode 100644 index f6f4abe41..000000000 --- a/.changeset/shaggy-jars-brake.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/runtime-manager': patch ---- - -Update runtime manager to handle workflows and mach new design diff --git a/.changeset/short-pens-punch.md b/.changeset/short-pens-punch.md deleted file mode 100644 index d5363759f..000000000 --- a/.changeset/short-pens-punch.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/runtime': patch ---- - -Support a cacheKey to bust cached modules in long-running processes diff --git a/.changeset/tough-coats-unite.md b/.changeset/tough-coats-unite.md deleted file mode 100644 index 860937bde..000000000 --- a/.changeset/tough-coats-unite.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/runtime': patch ---- - -Trigger callbacks on job start, complete and init diff --git a/.changeset/warm-tables-explode.md b/.changeset/warm-tables-explode.md deleted file mode 100644 index c4c215802..000000000 --- a/.changeset/warm-tables-explode.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/logger': patch ---- - -Add proxy function" diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md new file mode 100644 index 000000000..f1b293947 --- /dev/null +++ b/integration-tests/worker/CHANGELOG.md @@ -0,0 +1,12 @@ +# @openfn/integration-tests-worker + +## 1.0.1 + +### Patch Changes + +- Updated dependencies [7e4529e] +- Updated dependencies [1b6fa8e] + - @openfn/logger@0.0.18 + - @openfn/engine-multi@0.1.1 + - @openfn/lightning-mock@1.0.1 + - @openfn/ws-worker@0.1.1 diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index 83d8c6bfa..524a41d25 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.0", + "version": "1.0.1", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c174da7b7..eb374d815 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -389,31 +389,19 @@ importers: '@openfn/runtime': specifier: workspace:* version: link:../runtime - koa: - specifier: ^2.13.4 - version: 2.13.4 workerpool: specifier: ^6.5.1 version: 6.5.1 devDependencies: - '@types/koa': - specifier: ^2.13.5 - version: 2.13.5 '@types/node': specifier: ^18.15.13 version: 18.15.13 - '@types/nodemon': - specifier: ^1.19.2 - version: 1.19.2 '@types/workerpool': specifier: ^6.4.4 version: 6.4.4 ava: specifier: 5.3.1 version: 5.3.1 - nodemon: - specifier: ^2.0.19 - version: 2.0.19 ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@18.15.13)(typescript@5.1.6) @@ -662,12 +650,6 @@ importers: '@types/koa-bodyparser': specifier: ^4.3.10 version: 4.3.10 - '@types/koa-route': - specifier: ^3.2.6 - version: 3.2.6 - '@types/koa-websocket': - specifier: ^5.0.8 - version: 5.0.8 '@types/koa__router': specifier: ^12.0.1 version: 12.0.1 @@ -686,18 +668,9 @@ importers: ava: specifier: 5.1.0 version: 5.1.0 - koa-route: - specifier: ^3.2.0 - version: 3.2.0 - koa-websocket: - specifier: ^7.0.0 - version: 7.0.0 nodemon: specifier: 3.0.1 version: 3.0.1 - query-string: - specifier: ^8.1.0 - version: 8.1.0 ts-node: specifier: ^10.9.1 version: 10.9.1(@types/node@18.15.13)(typescript@4.6.4) @@ -1909,12 +1882,6 @@ packages: /@types/node@20.4.5: resolution: {integrity: sha512-rt40Nk13II9JwQBdeYqmbn2Q6IVTA5uPhvSO+JVqdXw/6/4glI6oR9ezty/A9Hg5u7JH4OmYmuQ+XvjKm0Datg==} - /@types/nodemon@1.19.2: - resolution: {integrity: sha512-4GWiTN3HevkxMIxEQ7OpD3MAHhlVsX2tairCMRmf8oYZxmhHw9+UpQpIdGdJrjsMT2Ty26FtJzUUcP/qM5fR8A==} - dependencies: - '@types/node': 18.15.13 - dev: true - /@types/nodemon@1.19.3: resolution: {integrity: sha512-LcKdWgch8uHOF73yYpdE7YPVLT0HnFI60zyNBpJyfAiDDwPy3WAxReQeB84UseE8e8qdJsBqmFXWbjxv7jlXBg==} dependencies: @@ -5784,24 +5751,6 @@ packages: whatwg-url: 5.0.0 dev: false - /nodemon@2.0.19: - resolution: {integrity: sha512-4pv1f2bMDj0Eeg/MhGqxrtveeQ5/G/UVe9iO6uTZzjnRluSA4PVWf8CW99LUPwGB3eNIA7zUFoP77YuI7hOc0A==} - engines: {node: '>=8.10.0'} - hasBin: true - requiresBuild: true - dependencies: - chokidar: 3.5.3 - debug: 3.2.7(supports-color@5.5.0) - ignore-by-default: 1.0.1 - minimatch: 3.1.2 - pstree.remy: 1.1.8 - semver: 5.7.2 - simple-update-notifier: 1.0.7 - supports-color: 5.5.0 - touch: 3.1.0 - undefsafe: 2.0.5 - dev: true - /nodemon@3.0.1: resolution: {integrity: sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==} engines: {node: '>=10'} @@ -6826,11 +6775,6 @@ packages: hasBin: true dev: true - /semver@7.0.0: - resolution: {integrity: sha512-+GB6zVA9LWh6zovYQLALHwv5rb2PHGlJi3lfiqIHxR0uuwCgefcOJc59v9fv1w8GbStwxuuqqAjI9NMAOOgq1A==} - hasBin: true - dev: true - /semver@7.5.4: resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} engines: {node: '>=10'} @@ -6940,13 +6884,6 @@ packages: resolution: {integrity: sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==} engines: {node: '>=14'} - /simple-update-notifier@1.0.7: - resolution: {integrity: sha512-BBKgR84BJQJm6WjWFMHgLVuo61FBDSj1z/xSFUIozqO6wO7ii0JxCqlIud7Enr/+LhlbNI0whErq96P2qHNWew==} - engines: {node: '>=8.10.0'} - dependencies: - semver: 7.0.0 - dev: true - /simple-update-notifier@2.0.0: resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==} engines: {node: '>=10'}