From 40f6034995a9ad730c31bcc427bce923f3de0856 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 15:41:22 +0000 Subject: [PATCH 01/20] package lock --- pnpm-lock.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7940cf52d..5f71e6720 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,19 +146,19 @@ importers: specifier: ^5.1.6 version: 5.1.6 - integration-tests/worker/tmp/openfn/repo/attempts: + integration-tests/worker/tmp/repo/attempts: dependencies: '@openfn/language-common_latest': specifier: npm:@openfn/language-common@^1.11.1 version: /@openfn/language-common@1.11.1 - integration-tests/worker/tmp/openfn/repo/exit-reason: + integration-tests/worker/tmp/repo/exit-reason: dependencies: '@openfn/language-common_latest': specifier: npm:@openfn/language-common@^1.11.1 version: /@openfn/language-common@1.11.1 - integration-tests/worker/tmp/openfn/repo/integration: + integration-tests/worker/tmp/repo/integration: dependencies: '@openfn/language-common_latest': specifier: npm:@openfn/language-common@^1.11.1 From d542aa92681cee58fbf876e10892082c4da6987d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 15:45:30 +0000 Subject: [PATCH 02/20] worker: leave attempt channel when finshed working --- .changeset/five-mirrors-smile.md | 5 +++++ packages/ws-worker/src/api/execute.ts | 14 +++++++------- packages/ws-worker/src/server.ts | 5 +++-- packages/ws-worker/test/api/execute.test.ts | 8 ++++---- packages/ws-worker/test/reasons.test.ts | 6 +++--- 5 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 .changeset/five-mirrors-smile.md diff --git a/.changeset/five-mirrors-smile.md b/.changeset/five-mirrors-smile.md new file mode 100644 index 000000000..ab69f192d --- /dev/null +++ b/.changeset/five-mirrors-smile.md @@ -0,0 +1,5 @@ +--- +'@openfn/ws-worker': patch +--- + +worker: leave attempt channel when finished working diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 311619f39..b015dcca7 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -35,7 +35,7 @@ export type Context = { channel: Channel; state: AttemptState; logger: Logger; - onComplete: (result: any) => void; + onFinish: (result: any) => void; }; // mapping engine events to lightning events @@ -55,13 +55,13 @@ export function execute( logger: Logger, plan: ExecutionPlan, options: AttemptOptions = {}, - onComplete = (_result: any) => {} + onFinish = (_result: any) => {} ) { logger.info('executing ', plan.id); const state = createAttemptState(plan, options); - const context: Context = { channel, state, logger, onComplete }; + const context: Context = { channel, state, logger, onFinish }; type EventHandler = (context: any, event: any) => void; @@ -253,7 +253,7 @@ export function onWorkflowStart( } export async function onWorkflowComplete( - { state, channel, onComplete }: Context, + { state, channel, onFinish }: Context, _event: WorkflowCompletePayload ) { // TODO I dont think the attempt final dataclip IS the last job dataclip @@ -264,14 +264,14 @@ export async function onWorkflowComplete( final_dataclip_id: state.lastDataclipId!, ...reason, }); - onComplete({ reason, state: result }); + onFinish({ reason, state: result }); } // On error, for now, we just post to workflow complete // No unit tests on this (not least because I think it'll change soon) // NB this is a crash state! export async function onWorkflowError( - { state, channel, onComplete }: Context, + { state, channel, onFinish }: Context, event: WorkflowErrorPayload ) { // Should we not just report this reason? @@ -285,7 +285,7 @@ export async function onWorkflowError( ...reason, }); - onComplete({ reason }); + onFinish({ reason }); } export function onJobLog({ channel, state }: Context, event: JSONLog) { diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts index 735c5f802..284237ecb 100644 --- a/packages/ws-worker/src/server.ts +++ b/packages/ws-worker/src/server.ts @@ -149,8 +149,9 @@ function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { } = await joinAttemptChannel(app.socket, token, id, logger); // Callback to be triggered when the work is done (including errors) - const onComplete = () => { + const onFinish = () => { delete app.workflows[id]; + attemptChannel.leave(); }; const context = execute( attemptChannel, @@ -158,7 +159,7 @@ function createServer(engine: RuntimeEngine, options: ServerOptions = {}) { logger, plan, options, - onComplete + onFinish ); app.workflows[id] = context; diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index 1bf98c1b1..e2b668521 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -340,11 +340,11 @@ test('workflowComplete should send an attempt:complete event', async (t) => { const event = {}; - const context = { channel, state, onComplete: () => {} }; + const context = { channel, state, onFinish: () => {} }; await onWorkflowComplete(context, event); }); -test('workflowComplete should call onComplete with final dataclip', async (t) => { +test('workflowComplete should call onFinish with final dataclip', async (t) => { const result = { answer: 42 }; const state = { @@ -362,7 +362,7 @@ test('workflowComplete should call onComplete with final dataclip', async (t) => const context = { channel, state, - onComplete: ({ state: finalState }) => { + onFinish: ({ state: finalState }) => { t.deepEqual(result, finalState); }, }; @@ -398,7 +398,7 @@ test('loadCredential should fetch a credential', async (t) => { t.deepEqual(state, { apiKey: 'abc' }); }); -test('execute should pass the final result to onComplete', async (t) => { +test('execute should pass the final result to onFinish', async (t) => { const channel = mockChannel(mockEventHandlers); const engine = await createMockRTE(); const logger = createMockLogger(); diff --git a/packages/ws-worker/test/reasons.test.ts b/packages/ws-worker/test/reasons.test.ts index 527c92af7..e7b9c4403 100644 --- a/packages/ws-worker/test/reasons.test.ts +++ b/packages/ws-worker/test/reasons.test.ts @@ -16,7 +16,7 @@ import { import { ExitReason } from '../src/types'; // Explicit tests of exit reasons coming out of the worker -// these test the onComplete callback +// these test the onFinish callback // uses the real runtime engine let engine; @@ -48,12 +48,12 @@ const execute = async (plan) => [ATTEMPT_COMPLETE]: async () => true, }); - const onComplete = (result) => { + const onFinish = (result) => { done(result); }; // @ts-ignore - doExecute(channel, engine, logger, plan, {}, onComplete); + doExecute(channel, engine, logger, plan, {}, onFinish); }); test('success', async (t) => { From 857c42ba99a6c26869de158766166f691007dff6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 15:53:51 +0000 Subject: [PATCH 03/20] runtime: adjust log output for job duration --- .changeset/old-mails-wash.md | 5 +++++ packages/runtime/src/execute/job.ts | 7 ++++--- packages/runtime/test/execute/job.test.ts | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 .changeset/old-mails-wash.md diff --git a/.changeset/old-mails-wash.md b/.changeset/old-mails-wash.md new file mode 100644 index 000000000..ef3a9b4fb --- /dev/null +++ b/.changeset/old-mails-wash.md @@ -0,0 +1,5 @@ +--- +'@openfn/runtime': patch +--- + +Fix log output for job duration diff --git a/packages/runtime/src/execute/job.ts b/packages/runtime/src/execute/job.ts index cb58b9594..5db9c70ce 100644 --- a/packages/runtime/src/execute/job.ts +++ b/packages/runtime/src/execute/job.ts @@ -110,7 +110,8 @@ const executeJob = async ( // We should by this point have validated the plan, so the job MUST exist - logger.timer('job'); + const timerId = `job-${jobId}`; + logger.timer(timerId); logger.always('Starting job', jobId); // The expression SHOULD return state, but COULD return anything @@ -123,7 +124,7 @@ const executeJob = async ( // TODO include the upstream job notify(NOTIFY_JOB_START, { jobId }); result = await executeExpression(ctx, job.expression, state); - const humanDuration = logger.timer('job'); + const humanDuration = logger.timer(timerId); logger.success(`Completed job ${jobId} in ${humanDuration}`); } catch (e: any) { didError = true; @@ -133,7 +134,7 @@ const executeJob = async ( // Whatever the final state was, save that as the intial state to the next thing result = state; - const duration = logger.timer('job'); + const duration = logger.timer(timerId); logger.error(`Failed job ${jobId} after ${duration}`); report(state, jobId, error); diff --git a/packages/runtime/test/execute/job.test.ts b/packages/runtime/test/execute/job.test.ts index f211eda67..972fa7b31 100644 --- a/packages/runtime/test/execute/job.test.ts +++ b/packages/runtime/test/execute/job.test.ts @@ -239,3 +239,18 @@ test(`notify ${NOTIFY_JOB_ERROR} for a fail`, async (t) => { await execute(context, job, state); }); + +test('log duration of execution', async (t) => { + const job = { + id: 'j', + expression: [(s: State) => s], + }; + const initialState = createState(); + const context = createContext(); + + await execute(context, job, initialState); + + const duration = logger._find('success', /completed job /i); + + t.regex(duration?.message, /completed job j in \d\d?ms/i); +}); From 60b036fdf8968ff67002377e4c0dd6bd7bc82632 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 16:51:41 +0000 Subject: [PATCH 04/20] engine: better error handling on autoinstall errors --- packages/engine-multi/src/api/autoinstall.ts | 11 ++++++++++- packages/engine-multi/src/api/execute.ts | 10 +++++----- packages/engine-multi/src/errors.ts | 13 ++++++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index eb47defd7..5ea734d6e 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -12,6 +12,7 @@ import { install as runtimeInstall } from '@openfn/runtime'; import type { Logger } from '@openfn/logger'; import type { ExecutionContext } from '../types'; import { AUTOINSTALL_COMPLETE, AUTOINSTALL_ERROR } from '../events'; +import { AutoinstallError } from '../errors'; // none of these options should be on the plan actually export type AutoinstallOptions = { @@ -54,6 +55,7 @@ const autoinstall = async (context: ExecutionContext): Promise => { // TODO would rather do all this in parallel but this is fine for now // TODO set iteration is weirdly difficult? const paths: ModulePaths = {}; + for (const a of adaptors) { // Ensure that this is not blacklisted // TODO what if it is? For now we'll log and skip it @@ -86,7 +88,11 @@ const autoinstall = async (context: ExecutionContext): Promise => { }); delete pending[a]; }) - .catch((e) => { + .catch((e: any) => { + delete pending[a]; + + logger.error(`ERROR autoinstalling ${a}: ${e.message}`); + logger.error(e); const duration = Date.now() - startTime; context.emit(AUTOINSTALL_ERROR, { module: name, @@ -94,6 +100,9 @@ const autoinstall = async (context: ExecutionContext): Promise => { duration, message: e.message || e.toString(), }); + + // wrap and re-throw the error + throw new AutoinstallError(a, e); }); } // Return the pending promise (safe to do this multiple times) diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 821887cd1..982bfa66e 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -99,11 +99,11 @@ const execute = async (context: ExecutionContext) => { error(context, { workflowId: state.plan.id, error: e }); logger.error(e); }); - } catch (e) { - // generic error wrapper - // this will catch anything you! - const wrappedError = new ExecutionError(e); - error(context, { workflowId: state.plan.id, error: wrappedError }); + } catch (e: any) { + if (!error.severity) { + e = new ExecutionError(e); + } + error(context, { workflowId: state.plan.id, error: e }); } }; diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts index 452108a13..c88fab172 100644 --- a/packages/engine-multi/src/errors.ts +++ b/packages/engine-multi/src/errors.ts @@ -57,5 +57,16 @@ export class CompileError extends EngineError { } } -// Autoinstall Error (exception) +export class AutoinstallError extends EngineError { + severity = 'exception'; // Syntax errors are crashes, but what if we get a module resolution thing? + type = 'AutoinstallError'; + message; + + constructor(specifier: string, error: any) { + super(); + + this.message = `Error installing ${specifier}: ${error.message}`; + } +} + // CredentialsError (exception) From f871dd3fc2ec5660b70e56b5ff966be36e5a604e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:12:55 +0000 Subject: [PATCH 05/20] engine: errors on autoinstall --- packages/engine-multi/src/errors.ts | 1 + .../engine-multi/test/api/autoinstall.test.ts | 113 +++++++++++++++++- 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts index c88fab172..34145d720 100644 --- a/packages/engine-multi/src/errors.ts +++ b/packages/engine-multi/src/errors.ts @@ -60,6 +60,7 @@ export class CompileError extends EngineError { export class AutoinstallError extends EngineError { severity = 'exception'; // Syntax errors are crashes, but what if we get a module resolution thing? type = 'AutoinstallError'; + name = 'AutoinstallError'; message; constructor(specifier: string, error: any) { diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index b2501ce47..2903b29f9 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -2,7 +2,7 @@ import test from 'ava'; import { createMockLogger } from '@openfn/logger'; import autoinstall, { identifyAdaptors } from '../../src/api/autoinstall'; -import { AUTOINSTALL_COMPLETE } from '../../src/events'; +import { AUTOINSTALL_COMPLETE, AUTOINSTALL_ERROR } from '../../src/events'; import ExecutionContext from '../../src/classes/ExecutionContext'; import whitelist from '../../src/whitelist'; @@ -271,4 +271,113 @@ test.serial('autoinstall: emit an event on completion', async (t) => { t.assert(event.duration >= 10); }); -test.todo('autoinstall: emit on error'); +test.serial('autoinstall: throw on error', async (t) => { + const mockIsInstalled = async () => false; + const mockInstall = async () => { + throw new Error('err'); + }; + + const autoinstallOpts = { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }; + const context = createContext(autoinstallOpts); + + await t.throwsAsync(() => autoinstall(context), { + name: 'AutoinstallError', + message: 'Error installing @openfn/language-common@1.0.0: err', + }); +}); + +test.serial('autoinstall: throw on error twice if pending', async (t) => { + return new Promise((done) => { + let callCount = 0; + let errCount = 0; + const mockIsInstalled = async () => false; + const mockInstall = async () => { + callCount++; + return new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('err')), 50); + }); + }; + + const autoinstallOpts = { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }; + const context = createContext(autoinstallOpts); + + autoinstall(context).catch((e) => { + t.is(e.name, 'AutoinstallError'); + errCount += 1; + }); + + autoinstall(context).catch((e) => { + errCount += 1; + t.is(e.name, 'AutoinstallError'); + t.is(callCount, 1); + t.is(errCount, 2); + t.pass('threw twice!'); + done(); + }); + }); +}); + +test.serial('autoinstall: emit on error', async (t) => { + let evt; + const mockIsInstalled = async () => false; + const mockInstall = async () => { + throw new Error('err'); + }; + + const autoinstallOpts = { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }; + const context = createContext(autoinstallOpts); + + context.on(AUTOINSTALL_ERROR, (e) => { + evt = e; + }); + + try { + await autoinstall(context); + } catch (e) { + // do nothing + } + + t.is(evt.module, '@openfn/language-common'); + t.is(evt.version, '1.0.0'); + t.is(evt.message, 'err'); + t.true(!isNaN(evt.duration)); +}); + +test.serial('autoinstall: throw twice in a ror', async (t) => { + let callCount = 0; + + const mockIsInstalled = async () => false; + const mockInstall = async () => { + callCount++; + return new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('err')), 1); + }); + }; + + const autoinstallOpts = { + handleInstall: mockInstall, + handleIsInstalled: mockIsInstalled, + }; + const context = createContext(autoinstallOpts); + + await t.throwsAsync(() => autoinstall(context), { + name: 'AutoinstallError', + message: 'Error installing @openfn/language-common@1.0.0: err', + }); + t.is(callCount, 1); + + await t.throwsAsync(() => autoinstall(context), { + name: 'AutoinstallError', + message: 'Error installing @openfn/language-common@1.0.0: err', + }); + t.is(callCount, 2); +}); From 36a3b7f151f00d465d1d2f4fa7b83548c732ab95 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:15:11 +0000 Subject: [PATCH 06/20] worker: more autoinstall logging --- packages/engine-multi/src/api/autoinstall.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/engine-multi/src/api/autoinstall.ts b/packages/engine-multi/src/api/autoinstall.ts index 5ea734d6e..881f34de7 100644 --- a/packages/engine-multi/src/api/autoinstall.ts +++ b/packages/engine-multi/src/api/autoinstall.ts @@ -73,8 +73,6 @@ const autoinstall = async (context: ExecutionContext): Promise => { const needsInstalling = !(await isInstalledFn(a, repoDir, logger)); if (needsInstalling) { if (!pending[a]) { - // TODO because autoinstall can take a while, we should emit that we're starting - // add a promise to the pending array const startTime = Date.now(); pending[a] = installFn(a, repoDir, logger) .then(() => { @@ -104,6 +102,10 @@ const autoinstall = async (context: ExecutionContext): Promise => { // wrap and re-throw the error throw new AutoinstallError(a, e); }); + } else { + logger.info( + `autoinstall waiting for previous promise for ${a} to resolve...` + ); } // Return the pending promise (safe to do this multiple times) // TODO if this is a chained promise, emit something like "using cache for ${name}" From f17fb4ae5e02d5af9c3d50b9fdbc6b435a3050a9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:15:17 +0000 Subject: [PATCH 07/20] changeset --- .changeset/selfish-spoons-sort.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/selfish-spoons-sort.md diff --git a/.changeset/selfish-spoons-sort.md b/.changeset/selfish-spoons-sort.md new file mode 100644 index 000000000..65796d7a2 --- /dev/null +++ b/.changeset/selfish-spoons-sort.md @@ -0,0 +1,5 @@ +--- +'@openfn/engine-multi': patch +--- + +Better error handling in autoinstall From 02fdafc907471211d8c46868be17112e018e1494 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:27:25 +0000 Subject: [PATCH 08/20] engine: refine autoinstall eror propagation --- packages/engine-multi/src/api/execute.ts | 2 +- packages/engine-multi/src/api/lifecycle.ts | 8 +++++++- packages/engine-multi/src/events.ts | 1 + packages/engine-multi/src/worker/events.ts | 5 +++-- packages/engine-multi/test/api/autoinstall.test.ts | 3 ++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts index 982bfa66e..2a4669b24 100644 --- a/packages/engine-multi/src/api/execute.ts +++ b/packages/engine-multi/src/api/execute.ts @@ -100,7 +100,7 @@ const execute = async (context: ExecutionContext) => { logger.error(e); }); } catch (e: any) { - if (!error.severity) { + if (!e.severity) { e = new ExecutionError(e); } error(context, { workflowId: state.plan.id, error: e }); diff --git a/packages/engine-multi/src/api/lifecycle.ts b/packages/engine-multi/src/api/lifecycle.ts index dfaf8aacb..234204e72 100644 --- a/packages/engine-multi/src/api/lifecycle.ts +++ b/packages/engine-multi/src/api/lifecycle.ts @@ -135,12 +135,18 @@ export const log = ( }); }; -export const error = (context: ExecutionContext, event: any) => { +export const error = ( + context: ExecutionContext, + event: internalEvents.ErrorEvent +) => { const { threadId = '-', error } = event; context.emit(externalEvents.WORKFLOW_ERROR, { threadId, + // @ts-ignore type: error.type || error.name || 'ERROR', message: error.message || error.toString(), + // default to exception because if we don't know, it's our fault + severity: error.severity || 'exception', }); }; diff --git a/packages/engine-multi/src/events.ts b/packages/engine-multi/src/events.ts index eca1cae54..b7bd6cdac 100644 --- a/packages/engine-multi/src/events.ts +++ b/packages/engine-multi/src/events.ts @@ -60,6 +60,7 @@ export interface WorkflowCompletePayload extends ExternalEvent { export interface WorkflowErrorPayload extends ExternalEvent { type: string; message: string; + severity: string; } export interface JobStartPayload extends ExternalEvent { diff --git a/packages/engine-multi/src/worker/events.ts b/packages/engine-multi/src/worker/events.ts index 8ae5fb60c..6ed599069 100644 --- a/packages/engine-multi/src/worker/events.ts +++ b/packages/engine-multi/src/worker/events.ts @@ -59,8 +59,9 @@ export interface LogEvent extends InternalEvent { message: JSONLog; } -export interface ErrorEvent extends InternalEvent { - jobId?: string; +export interface ErrorEvent { + threadId?: string; + workflowId?: string; error: { message: string; type: string; diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index 2903b29f9..b9846ab8d 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -289,6 +289,7 @@ test.serial('autoinstall: throw on error', async (t) => { }); }); +// TODO this is a bit flaky apparently test.serial('autoinstall: throw on error twice if pending', async (t) => { return new Promise((done) => { let callCount = 0; @@ -297,7 +298,7 @@ test.serial('autoinstall: throw on error twice if pending', async (t) => { const mockInstall = async () => { callCount++; return new Promise((_resolve, reject) => { - setTimeout(() => reject(new Error('err')), 50); + setTimeout(() => reject(new Error('err')), 100); }); }; From 93847729b84c57fe8c134eca2137f0ae568fac25 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:42:02 +0000 Subject: [PATCH 09/20] worker: more logging in error cases --- packages/ws-worker/src/api/execute.ts | 27 ++++++++++++++------- packages/ws-worker/test/api/execute.test.ts | 6 ++--- packages/ws-worker/test/reasons.test.ts | 27 +++++++++++++++++++++ 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index b015dcca7..291eae9dd 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -72,6 +72,11 @@ export function execute( // TODO for debugging and monitoring, we should also send events to the worker's event emitter const addEvent = (eventName: string, handler: EventHandler) => { const wrappedFn = async (event: any) => { + // TODO this logging is in the wrong place + // This actually logs errors coming out of the worker + // But it presents as logging from messages being send to lightning + // really this messaging should move into send event + // @ts-ignore const lightningEvent = eventMap[eventName] ?? eventName; try { @@ -271,21 +276,25 @@ export async function onWorkflowComplete( // No unit tests on this (not least because I think it'll change soon) // NB this is a crash state! export async function onWorkflowError( - { state, channel, onFinish }: Context, + { state, channel, logger, onFinish }: Context, event: WorkflowErrorPayload ) { // Should we not just report this reason? // Nothing more severe can have happened downstream, right? // const reason = calculateAttemptExitReason(state); + try { + // Ok, let's try that, let's just generate a reason from the event + const reason = calculateJobExitReason('', { data: {} }, event); + await sendEvent(channel, ATTEMPT_COMPLETE, { + final_dataclip_id: state.lastDataclipId!, + ...reason, + }); - // Ok, let's try that, let's just generate a reason from the event - const reason = calculateJobExitReason('', { data: {} }, event); - await sendEvent(channel, ATTEMPT_COMPLETE, { - final_dataclip_id: state.lastDataclipId!, - ...reason, - }); - - onFinish({ reason }); + onFinish({ reason }); + } catch (e: any) { + logger.error('ERROR in workflow-error handler:', e.message); + logger.error(e); + } } export function onJobLog({ channel, state }: Context, event: JSONLog) { diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts index e2b668521..d6af0e965 100644 --- a/packages/ws-worker/test/api/execute.test.ts +++ b/packages/ws-worker/test/api/execute.test.ts @@ -53,14 +53,14 @@ test('send event should resolve when the event is acknowledged', async (t) => { t.is(result, 22); }); -test('send event should throw if the event errors', async (t) => { +test('send event should throw if an event errors', async (t) => { const channel = mockChannel({ - echo: (x) => { + throw: (x) => { throw new Error('err'); }, }); - await t.throwsAsync(() => sendEvent(channel, 'echo', 22), { + await t.throwsAsync(() => sendEvent(channel, 'throw', 22), { message: 'err', }); }); diff --git a/packages/ws-worker/test/reasons.test.ts b/packages/ws-worker/test/reasons.test.ts index e7b9c4403..d84b5109c 100644 --- a/packages/ws-worker/test/reasons.test.ts +++ b/packages/ws-worker/test/reasons.test.ts @@ -30,6 +30,13 @@ test.before(async () => { maxWorkers: 1, purge: false, logger, + autoinstall: { + handleIsInstalled: async () => false, + handleInstall: () => + new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error('not the way to amarillo')), 1); + }), + }, }); }); @@ -198,6 +205,26 @@ test('crash: syntax error', async (t) => { t.is(reason.error_message, 'a: Unexpected token (1:2)'); }); +test('exception: autoinstall error', async (t) => { + const plan = createPlan({ + id: 'a', + expression: '.', + adaptor: '@openfn/language-common@1.0.0', + }); + + // TODO I also need to ensure that this calls attempt:complete + // I think that test lives elsewhere though + // I *think* I need to change the mock engine first though... + const { reason } = await execute(plan); + + t.is(reason.reason, 'exception'); + t.is(reason.error_type, 'AutoinstallError'); + t.is( + reason.error_message, + 'Error installing @openfn/language-common@1.0.0: not the way to amarillo' + ); +}); + test.todo('crash: workflow validation error'); test.todo('fail: adaptor error'); test.todo('crash: import error'); From 4317d6e0cee6095d38cb30501dbb7e77c813b06d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:47:32 +0000 Subject: [PATCH 10/20] tests: add autoinstall error --- .../worker/test/exit-reasons.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/integration-tests/worker/test/exit-reasons.test.ts b/integration-tests/worker/test/exit-reasons.test.ts index 8700fbeeb..0128c0b9f 100644 --- a/integration-tests/worker/test/exit-reasons.test.ts +++ b/integration-tests/worker/test/exit-reasons.test.ts @@ -53,3 +53,26 @@ test('crash: syntax error', async (t) => { t.is(error_type, 'CompileError'); t.regex(error_message, /Unexpected token \(1:9\)$/); }); + +test('exception: autoinstall error', async (t) => { + const attempt = { + id: crypto.randomUUID(), + jobs: [ + { + adaptor: '@openfn/language-worker-integration-tests@9.9.9', + body: 'fn((s) => s)', + }, + ], + }; + + const result = await run(attempt); + + const { reason, error_type, error_message } = result; + + t.is(reason, 'exception'); + t.is(error_type, 'AutoinstallError'); + t.regex( + error_message, + /Error installing @openfn\/language-worker-integration-tests@9.9.9/ + ); +}); From beb379c1e5dfbd5a53c28e3d2da79977f7b6ff8e Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:55:11 +0000 Subject: [PATCH 11/20] package lock --- pnpm-lock.yaml | 510 ++----------------------------------------------- 1 file changed, 12 insertions(+), 498 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5f71e6720..66a8db998 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,26 +146,7 @@ importers: specifier: ^5.1.6 version: 5.1.6 - integration-tests/worker/tmp/repo/attempts: - dependencies: - '@openfn/language-common_latest': - specifier: npm:@openfn/language-common@^1.11.1 - version: /@openfn/language-common@1.11.1 - - integration-tests/worker/tmp/repo/exit-reason: - dependencies: - '@openfn/language-common_latest': - specifier: npm:@openfn/language-common@^1.11.1 - version: /@openfn/language-common@1.11.1 - - integration-tests/worker/tmp/repo/integration: - dependencies: - '@openfn/language-common_latest': - specifier: npm:@openfn/language-common@^1.11.1 - version: /@openfn/language-common@1.11.1 - '@openfn/language-http_5.0.4': - specifier: npm:@openfn/language-http@^5.0.4 - version: /@openfn/language-http@5.0.4 + integration-tests/worker/tmp/repo/exit-reason: {} packages/cli: dependencies: @@ -1351,11 +1332,6 @@ packages: heap: 0.2.7 dev: false - /@fastify/busboy@2.1.0: - resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==} - engines: {node: '>=14'} - dev: false - /@inquirer/checkbox@1.3.5: resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==} engines: {node: '>=14.18.0'} @@ -1621,21 +1597,6 @@ packages: semver: 7.5.4 dev: true - /@openfn/language-common@1.11.1: - resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==} - dependencies: - ajv: 8.12.0 - axios: 1.1.3 - csv-parse: 5.5.2 - csvtojson: 2.0.10 - date-fns: 2.30.0 - jsonpath-plus: 4.0.0 - lodash: 4.17.21 - undici: 5.27.2 - transitivePeerDependencies: - - debug - dev: false - /@openfn/language-common@1.7.5: resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==} dependencies: @@ -1651,22 +1612,6 @@ packages: resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==} bundledDependencies: [] - /@openfn/language-http@5.0.4: - resolution: {integrity: sha512-zuMlJyORxBps0KO+93a3kVBRzStGwYVNAOEl7GgvO6Z96YBO7/K2NiqSyMHTyQeIyCLUBGmrEv1AAuk4pXxKAg==} - dependencies: - '@openfn/language-common': 1.11.1 - cheerio: 1.0.0-rc.12 - cheerio-tableparser: 1.0.1 - csv-parse: 4.16.3 - fast-safe-stringify: 2.1.1 - form-data: 3.0.1 - lodash: 4.17.21 - request: 2.88.2 - tough-cookie: 4.1.3 - transitivePeerDependencies: - - debug - dev: false - /@pkgjs/parseargs@0.11.0: resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -2113,24 +2058,6 @@ packages: clean-stack: 4.2.0 indent-string: 5.0.0 - /ajv@6.12.6: - resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} - dependencies: - fast-deep-equal: 3.1.3 - fast-json-stable-stringify: 2.1.0 - json-schema-traverse: 0.4.1 - uri-js: 4.4.1 - dev: false - - /ajv@8.12.0: - resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} - dependencies: - fast-deep-equal: 3.1.3 - json-schema-traverse: 1.0.0 - require-from-string: 2.0.2 - uri-js: 4.4.1 - dev: false - /ansi-colors@4.1.3: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} @@ -2266,17 +2193,6 @@ packages: resolution: {integrity: sha512-tLkvA81vQG/XqE2mjDkGQHoOINtMHtysSnemrmoGe6PydDPMRbVugqyk4A6V/WDWEfm3l+0d8anA9r8cv/5Jaw==} engines: {node: '>=12'} - /asn1@0.2.6: - resolution: {integrity: sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==} - dependencies: - safer-buffer: 2.1.2 - dev: false - - /assert-plus@1.0.0: - resolution: {integrity: sha512-NfJ4UzBCcQGLDlQq7nHxH+tv3kyZ0hHQqF5BO6J7tNJeP5do1llPr8dZ8zHonfhAu0PHAdMkSo+8o0wxg9lZWw==} - engines: {node: '>=0.8'} - dev: false - /assign-symbols@1.0.0: resolution: {integrity: sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==} engines: {node: '>=0.10.0'} @@ -2302,6 +2218,7 @@ packages: /asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + dev: true /atob@2.1.2: resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==} @@ -2432,14 +2349,6 @@ packages: fast-glob: 3.3.1 dev: true - /aws-sign2@0.7.0: - resolution: {integrity: sha512-08kcGqnYf/YmjoRhfxyu+CLxBjUtHLXLXX/vUfx9l2LYzG3c1m61nrpyFUZI6zeS+Li/wWMMidD9KgrqtGq3mA==} - dev: false - - /aws4@1.12.0: - resolution: {integrity: sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==} - dev: false - /axios@0.27.2: resolution: {integrity: sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==} dependencies: @@ -2457,6 +2366,7 @@ packages: proxy-from-env: 1.1.0 transitivePeerDependencies: - debug + dev: true /b4a@1.6.1: resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==} @@ -2493,12 +2403,6 @@ packages: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} dev: true - /bcrypt-pbkdf@1.0.2: - resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} - dependencies: - tweetnacl: 0.14.5 - dev: false - /bcryptjs@2.4.3: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: true @@ -2543,17 +2447,9 @@ packages: readable-stream: 4.2.0 dev: true - /bluebird@3.7.2: - resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} - dev: false - /blueimp-md5@2.19.0: resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==} - /boolbase@1.0.0: - resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} - dev: false - /brace-expansion@1.1.11: resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} dependencies: @@ -2738,10 +2634,6 @@ packages: engines: {node: '>=6'} dev: true - /caseless@0.12.0: - resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} - dev: false - /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -2775,34 +2667,6 @@ packages: /chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} - /cheerio-select@2.1.0: - resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} - dependencies: - boolbase: 1.0.0 - css-select: 5.1.0 - css-what: 6.1.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - dev: false - - /cheerio-tableparser@1.0.1: - resolution: {integrity: sha512-SCSWdMoFvIue0jdFZqRNPXDCZ67vuirJEG3pfh3AAU2hwxe/qh1EQUkUNPWlZhd6DMjRlTfcpcPWbaowjwRnNQ==} - dev: false - - /cheerio@1.0.0-rc.12: - resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} - engines: {node: '>= 6'} - dependencies: - cheerio-select: 2.1.0 - dom-serializer: 2.0.0 - domhandler: 5.0.3 - domutils: 3.1.0 - htmlparser2: 8.0.2 - parse5: 7.1.2 - parse5-htmlparser2-tree-adapter: 7.0.0 - dev: false - /chokidar@2.1.8: resolution: {integrity: sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==} deprecated: Chokidar 2 does not receive security updates since 2019. Upgrade to chokidar 3 with 15x fewer dependencies @@ -2979,6 +2843,7 @@ packages: engines: {node: '>= 0.8'} dependencies: delayed-stream: 1.0.0 + dev: true /commander@4.1.1: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} @@ -3053,10 +2918,6 @@ packages: resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} dev: false - /core-util-is@1.0.2: - resolution: {integrity: sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ==} - dev: false - /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -3107,21 +2968,6 @@ packages: which: 2.0.2 dev: true - /css-select@5.1.0: - resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} - dependencies: - boolbase: 1.0.0 - css-what: 6.1.0 - domhandler: 5.0.3 - domutils: 3.1.0 - nth-check: 2.1.1 - dev: false - - /css-what@6.1.0: - resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} - engines: {node: '>= 6'} - dev: false - /cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} @@ -3138,10 +2984,7 @@ packages: /csv-parse@4.16.3: resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==} - - /csv-parse@5.5.2: - resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==} - dev: false + dev: true /csv-stringify@5.6.5: resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==} @@ -3157,29 +3000,12 @@ packages: stream-transform: 2.1.3 dev: true - /csvtojson@2.0.10: - resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==} - engines: {node: '>=4.0.0'} - hasBin: true - dependencies: - bluebird: 3.7.2 - lodash: 4.17.21 - strip-bom: 2.0.0 - dev: false - /currently-unhandled@0.4.1: resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==} engines: {node: '>=0.10.0'} dependencies: array-find-index: 1.0.2 - /dashdash@1.14.1: - resolution: {integrity: sha512-jRFi8UDGo6j+odZiEpjazZaWqEal3w/basFjQHQEwVtZJGDpxbH1MeYluwCS8Xq5wmLJooDlMgvVarmWfGM44g==} - engines: {node: '>=0.10'} - dependencies: - assert-plus: 1.0.0 - dev: false - /date-fns@2.30.0: resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} engines: {node: '>=0.11'} @@ -3314,6 +3140,7 @@ packages: /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dev: true /delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -3364,33 +3191,6 @@ packages: resolution: {integrity: sha512-+HlytyjlPKnIG8XuRG8WvmBP8xs8P71y+SKKS6ZXWoEgLuePxtDoUEiH7WkdePWrQ5JBpE6aoVqfZfJUQkjXwA==} dev: true - /dom-serializer@2.0.0: - resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - entities: 4.5.0 - dev: false - - /domelementtype@2.3.0: - resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} - dev: false - - /domhandler@5.0.3: - resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} - engines: {node: '>= 4'} - dependencies: - domelementtype: 2.3.0 - dev: false - - /domutils@3.1.0: - resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} - dependencies: - dom-serializer: 2.0.0 - domelementtype: 2.3.0 - domhandler: 5.0.3 - dev: false - /dreamopt@0.8.0: resolution: {integrity: sha512-vyJTp8+mC+G+5dfgsY+r3ckxlz+QMX40VjPQsZc5gxVAxLmi64TBoVkP54A/pRAXMXsbu2GMMBrZPxNv23waMg==} engines: {node: '>=0.4.0'} @@ -3414,13 +3214,6 @@ packages: /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - /ecc-jsbn@0.1.2: - resolution: {integrity: sha512-eh9O+hwRHNbG4BLTjEl3nw044CkGm5X6LoaCf7LPp7UU8Qrt47JYNi6nPX8xjW97TKGKm1ouctg0QSpZe9qrnw==} - dependencies: - jsbn: 0.1.1 - safer-buffer: 2.1.2 - dev: false - /ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} @@ -3459,11 +3252,6 @@ packages: ansi-colors: 4.1.3 dev: true - /entities@4.5.0: - resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} - engines: {node: '>=0.12'} - dev: false - /err-code@2.0.3: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} dev: true @@ -4150,10 +3938,6 @@ packages: is-extendable: 1.0.1 dev: true - /extend@3.0.2: - resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} - dev: false - /extendable-error@0.1.7: resolution: {integrity: sha512-UOiS2in6/Q0FK0R0q6UY9vYpQ21mr/Qn1KOnte7vsACuNJf514WvCCUHSRCPcgjPT2bAhNIJdlE6bVap1GKmeg==} dev: true @@ -4182,15 +3966,6 @@ packages: - supports-color dev: true - /extsprintf@1.3.0: - resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} - engines: {'0': node >=0.6.0} - dev: false - - /fast-deep-equal@3.1.3: - resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} - dev: false - /fast-diff@1.3.0: resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==} @@ -4223,10 +3998,6 @@ packages: resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==} dev: false - /fast-json-stable-stringify@2.1.0: - resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} - dev: false - /fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} dev: true @@ -4351,6 +4122,7 @@ packages: peerDependenciesMeta: debug: optional: true + dev: true /for-in@1.0.2: resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==} @@ -4365,19 +4137,6 @@ packages: signal-exit: 4.0.2 dev: true - /forever-agent@0.6.1: - resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} - dev: false - - /form-data@2.3.3: - resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} - engines: {node: '>= 0.12'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@2.5.1: resolution: {integrity: sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==} engines: {node: '>= 0.12'} @@ -4387,15 +4146,6 @@ packages: mime-types: 2.1.35 dev: true - /form-data@3.0.1: - resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} - engines: {node: '>= 6'} - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - mime-types: 2.1.35 - dev: false - /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -4403,6 +4153,7 @@ packages: asynckit: 0.4.0 combined-stream: 1.0.8 mime-types: 2.1.35 + dev: true /fragment-cache@0.2.1: resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==} @@ -4519,12 +4270,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /getpass@0.1.7: - resolution: {integrity: sha512-0fzj9JxOLfJ+XGLhR8ze3unN0KZCgZwiSSDz168VERjK8Wl8kVSdcu2kspd4s4wtAa1y/qrVRiAA0WclVsu0ng==} - dependencies: - assert-plus: 1.0.0 - dev: false - /glob-parent@3.1.0: resolution: {integrity: sha512-E8Ak/2+dZY6fnzlR7+ueWvhsH1SjHr4jjss4YS/h4py44jY9MhK/VFdaZJAWDz6BbL21KeteKxFSFpq8OS5gVA==} dependencies: @@ -4631,20 +4376,6 @@ packages: through2: 2.0.5 dev: true - /har-schema@2.0.0: - resolution: {integrity: sha512-Oqluz6zhGX8cyRaTQlFMPw80bSJVG2x/cFb8ZPhUILGgHka9SsokCCOQgpveePerqidZOrT14ipqfJb7ILcW5Q==} - engines: {node: '>=4'} - dev: false - - /har-validator@5.1.5: - resolution: {integrity: sha512-nmT2T0lljbxdQZfspsno9hgrG3Uir6Ks5afism62poxqBM6sDnMEuPmzTq8XN0OEwqKLLdh1jQI3qyE66Nzb3w==} - engines: {node: '>=6'} - deprecated: this library is no longer supported - dependencies: - ajv: 6.12.6 - har-schema: 2.0.0 - dev: false - /hard-rejection@2.1.0: resolution: {integrity: sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==} engines: {node: '>=6'} @@ -4730,15 +4461,6 @@ packages: lru-cache: 7.18.3 dev: true - /htmlparser2@8.0.2: - resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} - dependencies: - domelementtype: 2.3.0 - domhandler: 5.0.3 - domutils: 3.1.0 - entities: 4.5.0 - dev: false - /http-assert@1.5.0: resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} engines: {node: '>= 0.8'} @@ -4807,15 +4529,6 @@ packages: - supports-color dev: true - /http-signature@1.2.0: - resolution: {integrity: sha512-CAbnr6Rz4CYQkLYUtSNXxQPUH2gK8f3iWexVlsnMeD+GjlsQ0Xsy1cOX+mN3dtxYomRy21CiOzU8Uhw6OwncEQ==} - engines: {node: '>=0.8', npm: '>=1.3.7'} - dependencies: - assert-plus: 1.0.0 - jsprim: 1.4.2 - sshpk: 1.18.0 - dev: false - /https-proxy-agent@5.0.1: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} @@ -5214,10 +4927,6 @@ packages: has-symbols: 1.0.3 dev: true - /is-typedarray@1.0.0: - resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} - dev: false - /is-unicode-supported@0.1.0: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} @@ -5227,10 +4936,6 @@ packages: resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==} engines: {node: '>=12'} - /is-utf8@0.2.1: - resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==} - dev: false - /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -5271,10 +4976,6 @@ packages: engines: {node: '>=0.10.0'} dev: true - /isstream@0.1.2: - resolution: {integrity: sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==} - dev: false - /jackspeak@2.2.2: resolution: {integrity: sha512-mgNtVv4vUuaKA97yxUHoA3+FkuhtxkjdXEWOyB/N76fjy0FjezEt34oy3epBtvCvS+7DyKwqCFWx/oJLV5+kCg==} engines: {node: '>=14'} @@ -5315,10 +5016,6 @@ packages: argparse: 2.0.1 dev: true - /jsbn@0.1.1: - resolution: {integrity: sha512-UVU9dibq2JcFWxQPA6KCqj5O42VOmAY3zQUfEKxU0KpTGXwNoCjkX1e13eHNvw/xPynt6pU0rZ1htjWTNTSXsg==} - dev: false - /json-diff@1.0.6: resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==} hasBin: true @@ -5332,22 +5029,6 @@ packages: resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} dev: true - /json-schema-traverse@0.4.1: - resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} - dev: false - - /json-schema-traverse@1.0.0: - resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} - dev: false - - /json-schema@0.4.0: - resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} - dev: false - - /json-stringify-safe@5.0.1: - resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} - dev: false - /jsonfile@4.0.0: resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==} optionalDependencies: @@ -5362,6 +5043,7 @@ packages: /jsonpath-plus@4.0.0: resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==} engines: {node: '>=10.0'} + dev: true /jsonpath@1.1.1: resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==} @@ -5371,16 +5053,6 @@ packages: underscore: 1.12.1 dev: true - /jsprim@1.4.2: - resolution: {integrity: sha512-P2bSOMAc/ciLz6DzgjVlGJP9+BrJWu5UDGK70C2iweC5QBIeFf0ZXRvGjEj2uYgrY2MkAAhsSWHDWlFtEroZWw==} - engines: {node: '>=0.6.0'} - dependencies: - assert-plus: 1.0.0 - extsprintf: 1.3.0 - json-schema: 0.4.0 - verror: 1.10.0 - dev: false - /keygrip@1.1.0: resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} engines: {node: '>= 0.6'} @@ -6103,16 +5775,6 @@ packages: path-key: 3.1.1 dev: true - /nth-check@2.1.1: - resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} - dependencies: - boolbase: 1.0.0 - dev: false - - /oauth-sign@0.9.0: - resolution: {integrity: sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==} - dev: false - /object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -6382,19 +6044,6 @@ packages: resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==} engines: {node: '>=12'} - /parse5-htmlparser2-tree-adapter@7.0.0: - resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} - dependencies: - domhandler: 5.0.3 - parse5: 7.1.2 - dev: false - - /parse5@7.1.2: - resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} - dependencies: - entities: 4.5.0 - dev: false - /parseurl@1.3.3: resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} engines: {node: '>= 0.8'} @@ -6477,10 +6126,6 @@ packages: through2: 2.0.5 dev: true - /performance-now@2.1.0: - resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} - dev: false - /phoenix@1.7.7: resolution: {integrity: sha512-moAN6e4Z16x/x1nswUpnTR2v5gm7HsI7eluZ2YnYUUsBNzi3cY/5frmiJfXIEi877IQAafzTfp8hd6vEUMme+w==} dev: false @@ -6705,6 +6350,7 @@ packages: /proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + dev: true /proxy-middleware@0.15.0: resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==} @@ -6715,10 +6361,6 @@ packages: resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} dev: true - /psl@1.9.0: - resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} - dev: false - /pstree.remy@1.1.8: resolution: {integrity: sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==} dev: true @@ -6741,6 +6383,7 @@ packages: /punycode@2.3.0: resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==} engines: {node: '>=6'} + dev: true /qs@6.11.2: resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==} @@ -6749,11 +6392,6 @@ packages: side-channel: 1.0.4 dev: false - /qs@6.5.3: - resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} - engines: {node: '>=0.6'} - dev: false - /query-string@8.1.0: resolution: {integrity: sha512-BFQeWxJOZxZGix7y+SByG3F36dA0AbTy9o6pSmKFcFz7DAj0re9Frkty3saBn3nHo3D0oZJ/+rx3r8H8r8Jbpw==} engines: {node: '>=14.16'} @@ -6763,10 +6401,6 @@ packages: split-on-first: 3.0.0 dev: true - /querystringify@2.2.0: - resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==} - dev: false - /queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -6951,50 +6585,14 @@ packages: engines: {node: '>=0.10'} dev: true - /request@2.88.2: - resolution: {integrity: sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==} - engines: {node: '>= 6'} - deprecated: request has been deprecated, see https://github.com/request/request/issues/3142 - dependencies: - aws-sign2: 0.7.0 - aws4: 1.12.0 - caseless: 0.12.0 - combined-stream: 1.0.8 - extend: 3.0.2 - forever-agent: 0.6.1 - form-data: 2.3.3 - har-validator: 5.1.5 - http-signature: 1.2.0 - is-typedarray: 1.0.0 - isstream: 0.1.2 - json-stringify-safe: 5.0.1 - mime-types: 2.1.35 - oauth-sign: 0.9.0 - performance-now: 2.1.0 - qs: 6.5.3 - safe-buffer: 5.2.1 - tough-cookie: 2.5.0 - tunnel-agent: 0.6.0 - uuid: 3.4.0 - dev: false - /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} - /require-from-string@2.0.2: - resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} - engines: {node: '>=0.10.0'} - dev: false - /require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} dev: true - /requires-port@1.0.0: - resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} - dev: false - /resolve-cwd@3.0.0: resolution: {integrity: sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==} engines: {node: '>=8'} @@ -7406,22 +7004,6 @@ packages: /sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} - /sshpk@1.18.0: - resolution: {integrity: sha512-2p2KJZTSqQ/I3+HX42EpYOa2l3f8Erv8MWKsy2I9uf4wA7yFIkXRffYdsx86y6z4vHtV8u7g+pPlr8/4ouAxsQ==} - engines: {node: '>=0.10.0'} - hasBin: true - dependencies: - asn1: 0.2.6 - assert-plus: 1.0.0 - bcrypt-pbkdf: 1.0.2 - dashdash: 1.14.1 - ecc-jsbn: 0.1.2 - getpass: 0.1.7 - jsbn: 0.1.1 - safer-buffer: 2.1.2 - tweetnacl: 0.14.5 - dev: false - /ssri@10.0.4: resolution: {integrity: sha512-12+IR2CB2C28MMAw0Ncqwj5QbTcs0nGIhgJzYWzDkb21vWmfNI83KS4f3Ci6GI98WreIfG7o9UXp3C0qbpA8nQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7530,13 +7112,6 @@ packages: dependencies: ansi-regex: 6.0.1 - /strip-bom@2.0.0: - resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==} - engines: {node: '>=0.10.0'} - dependencies: - is-utf8: 0.2.1 - dev: false - /strip-bom@3.0.0: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} @@ -7753,24 +7328,6 @@ packages: nopt: 1.0.10 dev: true - /tough-cookie@2.5.0: - resolution: {integrity: sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==} - engines: {node: '>=0.8'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - dev: false - - /tough-cookie@4.1.3: - resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==} - engines: {node: '>=6'} - dependencies: - psl: 1.9.0 - punycode: 2.3.0 - universalify: 0.2.0 - url-parse: 1.5.10 - dev: false - /tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} dev: false @@ -8030,16 +7587,6 @@ packages: yargs: 17.7.2 dev: true - /tunnel-agent@0.6.0: - resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} - dependencies: - safe-buffer: 5.2.1 - dev: false - - /tweetnacl@0.14.5: - resolution: {integrity: sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==} - dev: false - /type-check@0.3.2: resolution: {integrity: sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==} engines: {node: '>= 0.8.0'} @@ -8118,13 +7665,6 @@ packages: resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==} dev: true - /undici@5.27.2: - resolution: {integrity: sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==} - engines: {node: '>=14.0'} - dependencies: - '@fastify/busboy': 2.1.0 - dev: false - /union-value@1.0.1: resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==} engines: {node: '>=0.10.0'} @@ -8154,11 +7694,6 @@ packages: engines: {node: '>= 4.0.0'} dev: true - /universalify@0.2.0: - resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==} - engines: {node: '>= 4.0.0'} - dev: false - /unix-crypt-td-js@1.1.4: resolution: {integrity: sha512-8rMeVYWSIyccIJscb9NdCfZKSRBKYTeVnwmiRYT2ulE3qd1RaDQ0xQDP+rI3ccIWbhu/zuo5cgN8z73belNZgw==} dev: true @@ -8180,12 +7715,6 @@ packages: engines: {node: '>=4'} dev: true - /uri-js@4.4.1: - resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} - dependencies: - punycode: 2.3.0 - dev: false - /urix@0.1.0: resolution: {integrity: sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==} deprecated: Please see https://github.com/lydell/urix#deprecated @@ -8196,13 +7725,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: false - /url-parse@1.5.10: - resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==} - dependencies: - querystringify: 2.2.0 - requires-port: 1.0.0 - dev: false - /use@3.1.1: resolution: {integrity: sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==} engines: {node: '>=0.10.0'} @@ -8221,6 +7743,7 @@ packages: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} deprecated: Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details. hasBin: true + dev: true /v8-compile-cache-lib@3.0.1: resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==} @@ -8243,15 +7766,6 @@ packages: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} - /verror@1.10.0: - resolution: {integrity: sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw==} - engines: {'0': node >=0.6.0} - dependencies: - assert-plus: 1.0.0 - core-util-is: 1.0.2 - extsprintf: 1.3.0 - dev: false - /wcwidth@1.0.1: resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==} dependencies: From fa3aa3e9c094387a02a8c3078d5e8e36f279e05b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 17:55:21 +0000 Subject: [PATCH 12/20] worker: typings --- packages/ws-worker/src/api/execute.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts index 291eae9dd..5cf804bb8 100644 --- a/packages/ws-worker/src/api/execute.ts +++ b/packages/ws-worker/src/api/execute.ts @@ -140,6 +140,7 @@ export function execute( workflowId: plan.id!, message: e.message, type: e.type, + severity: e.severity, }); } }); From 93f4c3f4bd5692e3fd61f1a821c7cfa9c031df43 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 09:51:49 +0000 Subject: [PATCH 13/20] engine: make test more robust --- .../engine-multi/test/api/autoinstall.test.ts | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/engine-multi/test/api/autoinstall.test.ts b/packages/engine-multi/test/api/autoinstall.test.ts index b9846ab8d..e3cb73074 100644 --- a/packages/engine-multi/test/api/autoinstall.test.ts +++ b/packages/engine-multi/test/api/autoinstall.test.ts @@ -289,7 +289,6 @@ test.serial('autoinstall: throw on error', async (t) => { }); }); -// TODO this is a bit flaky apparently test.serial('autoinstall: throw on error twice if pending', async (t) => { return new Promise((done) => { let callCount = 0; @@ -298,7 +297,7 @@ test.serial('autoinstall: throw on error twice if pending', async (t) => { const mockInstall = async () => { callCount++; return new Promise((_resolve, reject) => { - setTimeout(() => reject(new Error('err')), 100); + setTimeout(() => reject(new Error('err')), 10); }); }; @@ -308,19 +307,23 @@ test.serial('autoinstall: throw on error twice if pending', async (t) => { }; const context = createContext(autoinstallOpts); - autoinstall(context).catch((e) => { - t.is(e.name, 'AutoinstallError'); - errCount += 1; - }); + autoinstall(context).catch(assertCatches); - autoinstall(context).catch((e) => { - errCount += 1; + autoinstall(context).catch(assertCatches); + + // The two catches won't neccessarily return in order + // (shrug asynchronous code?) + // So this catch-all callback will resolve the test when both + // promises have resolved + function assertCatches(e) { t.is(e.name, 'AutoinstallError'); - t.is(callCount, 1); - t.is(errCount, 2); - t.pass('threw twice!'); - done(); - }); + errCount += 1; + if (errCount === 2) { + t.is(callCount, 1); + t.pass('threw twice!'); + done(); + } + } }); }); From 1be1579f85d187ccd04f3320b63f8d016fe4e3ca Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 15 Nov 2023 18:13:06 +0000 Subject: [PATCH 14/20] worker: change purge rules The engine could be busy autoinstalling or compiling while workerpool is idle... so we have to move the purge logic --- packages/engine-multi/src/api/call-worker.ts | 21 ++++++++++---------- packages/engine-multi/src/engine.ts | 11 +++++++--- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 1733f3f24..521d9acc0 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -48,20 +48,19 @@ export default function initWorkers( promise.timeout(timeout); } - if (options.purge) { - promise.then(() => { - const { pendingTasks } = workers.stats(); - if (pendingTasks == 0) { - logger?.debug('Purging workers'); - api.emit(PURGE); - workers.terminate(); - } - }); - } - return promise; }; + // @ts-ignore + api.purge = () => { + const { pendingTasks } = workers.stats(); + if (pendingTasks == 0) { + logger?.debug('Purging workers'); + api.emit(PURGE); + workers.terminate(); + } + }; + // This will force termination instantly api.closeWorkers = () => { workers.terminate(true); diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index fe38cf5cd..0cbf48190 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -183,14 +183,19 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { delete deferredListeners[workflowId]; } - // execute(context); - // Run the execute on a timeout so that consumers have a chance // to register listeners setTimeout(() => { // TODO typing between the class and interface isn't right // @ts-ignore - execute(context); + execute(context).finally(() => { + delete contexts[workflowId]; + // @ts-ignore + if (Object.keys(contexts).length === 0) { + // @ts-ignore + engine.purge?.(); + } + }); }, 1); // hmm. Am I happy to pass the internal workflow state OUT of the handler? From 1fa2175d217f09ca05c11fac6b61f2358d278670 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 10:26:03 +0000 Subject: [PATCH 15/20] worker: fix tests & types --- packages/engine-multi/src/api/call-worker.ts | 10 ++-- packages/engine-multi/src/engine.ts | 8 ---- packages/engine-multi/src/types.ts | 1 + .../engine-multi/test/api/call-worker.test.ts | 13 +----- packages/engine-multi/test/engine.test.ts | 46 +++++++++++++++++++ 5 files changed, 53 insertions(+), 25 deletions(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index 521d9acc0..acb48efdd 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -23,7 +23,7 @@ type WorkerOptions = { // Adds a `callWorker` function to the API object, which will execute a task in a worker export default function initWorkers( - api: EngineAPI, + engine: EngineAPI, workerPath: string, options: WorkerOptions = {}, logger?: Logger @@ -31,7 +31,7 @@ export default function initWorkers( // TODO can we verify the worker path and throw if it's invalid? // workerpool won't complain if we give it a nonsense path const workers = createWorkers(workerPath, options); - api.callWorker = ( + engine.callWorker = ( task: string, args: any[] = [], events: any = {}, @@ -52,17 +52,17 @@ export default function initWorkers( }; // @ts-ignore - api.purge = () => { + engine.purge = () => { const { pendingTasks } = workers.stats(); if (pendingTasks == 0) { logger?.debug('Purging workers'); - api.emit(PURGE); + engine.emit(PURGE); workers.terminate(); } }; // This will force termination instantly - api.closeWorkers = () => { + engine.closeWorkers = () => { workers.terminate(true); // Defer the return to allow workerpool to close down diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 0cbf48190..7c8af218f 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -98,11 +98,6 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { const contexts: Record = {}; const deferredListeners: Record[]> = {}; - // TODO I think this is for later - //const activeWorkflows: string[] = []; - - // TOOD I wonder if the engine should a) always accept a worker path - // and b) validate it before it runs let resolvedWorkerPath; if (workerPath) { // If a path to the worker has been passed in, just use it verbatim @@ -190,9 +185,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { // @ts-ignore execute(context).finally(() => { delete contexts[workflowId]; - // @ts-ignore if (Object.keys(contexts).length === 0) { - // @ts-ignore engine.purge?.(); } }); @@ -207,7 +200,6 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { context.once(evt, fn), off: (evt: string, fn: (...args: any[]) => void) => context.off(evt, fn), }; - // return context; }; const listen = ( diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts index 0d9f66f77..8dc5d76ea 100644 --- a/packages/engine-multi/src/types.ts +++ b/packages/engine-multi/src/types.ts @@ -64,6 +64,7 @@ export interface ExecutionContext extends EventEmitter { export interface EngineAPI extends EventEmitter { callWorker: CallWorker; closeWorkers: () => void; + purge?: () => void; } export interface RuntimeEngine extends EventEmitter { diff --git a/packages/engine-multi/test/api/call-worker.test.ts b/packages/engine-multi/test/api/call-worker.test.ts index 16b9897b1..1887d877c 100644 --- a/packages/engine-multi/test/api/call-worker.test.ts +++ b/packages/engine-multi/test/api/call-worker.test.ts @@ -12,7 +12,7 @@ let api = new EventEmitter() as EngineAPI; const workerPath = path.resolve('src/test/worker-functions.js'); test.before(() => { - initWorkers(api, workerPath, { purge: true }); + initWorkers(api, workerPath); }); test.after(() => api.closeWorkers()); @@ -78,17 +78,6 @@ test.serial('callWorker should execute in a different process', async (t) => { }); }); -test.serial('callWorker should try to purge workers on complete', async (t) => { - return new Promise((done) => { - api.on(PURGE, () => { - t.pass('purge event called'); - done(); - }); - - api.callWorker('test', []); - }); -}); - test.serial( 'If null env is passed, worker thread should be able to access parent env', async (t) => { diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index 44299c5c7..a2f3e2178 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -239,3 +239,49 @@ test.serial('timeout the whole attempt and emit an error', async (t) => { engine.execute(plan, opts); }); }); + +test.serial('Purge workers when a run is complete', async (t) => { + return new Promise(async (done) => { + const p = path.resolve('src/test/worker-functions.js'); + engine = await createEngine(options, p); + + const plan = { + id: 'a', + jobs: [ + { + expression: '34', + }, + ], + }; + + engine.on(e.PURGE, () => { + t.pass('purge event called'); + done(); + }); + + engine.execute(plan); + }); +}); + +test.serial('Purge workers when run errors', async (t) => { + return new Promise(async (done) => { + const p = path.resolve('src/test/worker-functions.js'); + engine = await createEngine(options, p); + + const plan = { + id: 'a', + jobs: [ + { + expression: 'throw new Error("test")', + }, + ], + }; + + engine.on(e.PURGE, () => { + t.pass('purge event called'); + done(); + }); + + engine.execute(plan); + }); +}); From 9406a0d5d7f7ddabcc3f93e9d966e52a01c10283 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 10:49:31 +0000 Subject: [PATCH 16/20] engine: respect purge option --- packages/engine-multi/src/engine.ts | 2 +- packages/engine-multi/test/engine.test.ts | 35 +++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts index 7c8af218f..c30a3d2c1 100644 --- a/packages/engine-multi/src/engine.ts +++ b/packages/engine-multi/src/engine.ts @@ -185,7 +185,7 @@ const createEngine = async (options: EngineOptions, workerPath?: string) => { // @ts-ignore execute(context).finally(() => { delete contexts[workflowId]; - if (Object.keys(contexts).length === 0) { + if (options.purge && Object.keys(contexts).length === 0) { engine.purge?.(); } }); diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index a2f3e2178..a70d057cc 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -22,6 +22,7 @@ const options = { autoinstall: { handleIsInstalled: async () => true, }, + purge: true, }; let engine; @@ -285,3 +286,37 @@ test.serial('Purge workers when run errors', async (t) => { engine.execute(plan); }); }); + +test.serial("Don't purge if purge is false", async (t) => { + return new Promise(async (done) => { + const p = path.resolve('src/test/worker-functions.js'); + engine = await createEngine( + { + ...options, + purge: false, + }, + p + ); + + const plan = { + id: 'a', + jobs: [ + { + expression: '34', + }, + ], + }; + + engine.on(e.PURGE, () => { + t.fail('purge event called'); + done(); + }); + + engine.execute(plan).on(e.WORKFLOW_COMPLETE, () => { + setTimeout(() => { + t.pass('no purge called within 50ms'); + done(); + }, 50); + }); + }); +}); From 7db08b5d589971cd8604ed951047c0ea4cd9bee7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 10:56:07 +0000 Subject: [PATCH 17/20] engine: remove ts-ignore --- packages/engine-multi/src/api/call-worker.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts index acb48efdd..952bc142f 100644 --- a/packages/engine-multi/src/api/call-worker.ts +++ b/packages/engine-multi/src/api/call-worker.ts @@ -51,7 +51,6 @@ export default function initWorkers( return promise; }; - // @ts-ignore engine.purge = () => { const { pendingTasks } = workers.stats(); if (pendingTasks == 0) { From a0893336acd229d12c88f229be7bc8a96086037d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 10:58:01 +0000 Subject: [PATCH 18/20] engine: relax timing in test --- packages/engine-multi/test/integration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine-multi/test/integration.test.ts b/packages/engine-multi/test/integration.test.ts index 03e9e8b84..8c7aa297e 100644 --- a/packages/engine-multi/test/integration.test.ts +++ b/packages/engine-multi/test/integration.test.ts @@ -83,7 +83,7 @@ test.serial('trigger job-complete', (t) => { api.execute(plan).on('job-complete', (evt) => { t.deepEqual(evt.next, []); - t.true(evt.duration < 10); + t.true(evt.duration < 20); t.is(evt.jobId, 'j1'); t.deepEqual(evt.state, { data: {} }); t.pass('job completed'); From 793d52364657182a77af8595c3eed59561b2f955 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 11:06:42 +0000 Subject: [PATCH 19/20] changeset --- .changeset/five-seals-divide.md | 5 +++++ packages/engine-multi/test/engine.test.ts | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .changeset/five-seals-divide.md diff --git a/.changeset/five-seals-divide.md b/.changeset/five-seals-divide.md new file mode 100644 index 000000000..ea4f61efe --- /dev/null +++ b/.changeset/five-seals-divide.md @@ -0,0 +1,5 @@ +--- +'@openfn/ws-engine': patch +--- + +Updated purge strategy diff --git a/packages/engine-multi/test/engine.test.ts b/packages/engine-multi/test/engine.test.ts index a70d057cc..35047d78e 100644 --- a/packages/engine-multi/test/engine.test.ts +++ b/packages/engine-multi/test/engine.test.ts @@ -1,7 +1,6 @@ import test from 'ava'; import path from 'node:path'; import { createMockLogger } from '@openfn/logger'; -import { createPlan } from '../src/test/util'; import createEngine from '../src/engine'; import * as e from '../src/events'; From 12bde0df888bf810dd36b30c3a676381cf121c8f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 16 Nov 2023 12:12:53 +0000 Subject: [PATCH 20/20] versions: worker@0.2.7 cli@2.4.8 --- .changeset/five-mirrors-smile.md | 5 ----- .changeset/five-seals-divide.md | 5 ----- .changeset/old-mails-wash.md | 5 ----- .changeset/selfish-spoons-sort.md | 5 ----- integration-tests/worker/CHANGELOG.md | 11 +++++++++++ integration-tests/worker/package.json | 2 +- packages/cli/CHANGELOG.md | 7 +++++++ packages/cli/package.json | 2 +- packages/engine-multi/CHANGELOG.md | 9 +++++++++ packages/engine-multi/package.json | 2 +- packages/lightning-mock/CHANGELOG.md | 10 ++++++++++ packages/lightning-mock/package.json | 2 +- packages/runtime/CHANGELOG.md | 6 ++++++ packages/runtime/package.json | 2 +- packages/ws-worker/CHANGELOG.md | 11 +++++++++++ packages/ws-worker/package.json | 2 +- pnpm-lock.yaml | 2 -- 17 files changed, 60 insertions(+), 28 deletions(-) delete mode 100644 .changeset/five-mirrors-smile.md delete mode 100644 .changeset/five-seals-divide.md delete mode 100644 .changeset/old-mails-wash.md delete mode 100644 .changeset/selfish-spoons-sort.md diff --git a/.changeset/five-mirrors-smile.md b/.changeset/five-mirrors-smile.md deleted file mode 100644 index ab69f192d..000000000 --- a/.changeset/five-mirrors-smile.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/ws-worker': patch ---- - -worker: leave attempt channel when finished working diff --git a/.changeset/five-seals-divide.md b/.changeset/five-seals-divide.md deleted file mode 100644 index ea4f61efe..000000000 --- a/.changeset/five-seals-divide.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/ws-engine': patch ---- - -Updated purge strategy diff --git a/.changeset/old-mails-wash.md b/.changeset/old-mails-wash.md deleted file mode 100644 index ef3a9b4fb..000000000 --- a/.changeset/old-mails-wash.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/runtime': patch ---- - -Fix log output for job duration diff --git a/.changeset/selfish-spoons-sort.md b/.changeset/selfish-spoons-sort.md deleted file mode 100644 index 65796d7a2..000000000 --- a/.changeset/selfish-spoons-sort.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/engine-multi': patch ---- - -Better error handling in autoinstall diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md index 7e7c5cab4..1d32859db 100644 --- a/integration-tests/worker/CHANGELOG.md +++ b/integration-tests/worker/CHANGELOG.md @@ -1,5 +1,16 @@ # @openfn/integration-tests-worker +## 1.0.17 + +### Patch Changes + +- Updated dependencies [d542aa9] +- Updated dependencies [793d523] +- Updated dependencies [f17fb4a] + - @openfn/ws-worker@0.2.7 + - @openfn/engine-multi@0.1.11 + - @openfn/lightning-mock@1.0.12 + ## 1.0.16 ### Patch Changes diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json index e3d8a8924..996c754fa 100644 --- a/integration-tests/worker/package.json +++ b/integration-tests/worker/package.json @@ -1,7 +1,7 @@ { "name": "@openfn/integration-tests-worker", "private": true, - "version": "1.0.16", + "version": "1.0.17", "description": "Lightning WOrker integration tests", "author": "Open Function Group ", "license": "ISC", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 99e781d65..232c98df7 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/cli +## 0.4.8 + +### Patch Changes + +- Updated dependencies [857c42b] + - @openfn/runtime@0.1.4 + ## 0.4.7 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index ded481688..a70c5763a 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "0.4.7", + "version": "0.4.8", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md index 0b4b88b69..64134dad1 100644 --- a/packages/engine-multi/CHANGELOG.md +++ b/packages/engine-multi/CHANGELOG.md @@ -1,5 +1,14 @@ # engine-multi +## 0.1.11 + +### Patch Changes + +- 793d523: Updated purge strategy +- f17fb4a: Better error handling in autoinstall +- Updated dependencies [857c42b] + - @openfn/runtime@0.1.4 + ## 0.1.10 ### Patch Changes diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json index 346c4b164..c5344df41 100644 --- a/packages/engine-multi/package.json +++ b/packages/engine-multi/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/engine-multi", - "version": "0.1.10", + "version": "0.1.11", "description": "Multi-process runtime engine", "main": "dist/index.js", "type": "module", diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md index b11a1ef37..252d9a97b 100644 --- a/packages/lightning-mock/CHANGELOG.md +++ b/packages/lightning-mock/CHANGELOG.md @@ -1,5 +1,15 @@ # @openfn/lightning-mock +## 1.0.12 + +### Patch Changes + +- Updated dependencies [793d523] +- Updated dependencies [857c42b] +- Updated dependencies [f17fb4a] + - @openfn/engine-multi@0.1.11 + - @openfn/runtime@0.1.4 + ## 1.0.11 ### Patch Changes diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json index 37563d355..3114c42fe 100644 --- a/packages/lightning-mock/package.json +++ b/packages/lightning-mock/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/lightning-mock", - "version": "1.0.11", + "version": "1.0.12", "private": true, "description": "A mock Lightning server", "main": "dist/index.js", diff --git a/packages/runtime/CHANGELOG.md b/packages/runtime/CHANGELOG.md index cc9021aa0..451706128 100644 --- a/packages/runtime/CHANGELOG.md +++ b/packages/runtime/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/runtime +## 0.1.4 + +### Patch Changes + +- 857c42b: Fix log output for job duration + ## 0.1.3 ### Patch Changes diff --git a/packages/runtime/package.json b/packages/runtime/package.json index b867b4ae9..aba0e2231 100644 --- a/packages/runtime/package.json +++ b/packages/runtime/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/runtime", - "version": "0.1.3", + "version": "0.1.4", "description": "Job processing runtime.", "type": "module", "exports": { diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md index 6d58c4d2b..5520fe65d 100644 --- a/packages/ws-worker/CHANGELOG.md +++ b/packages/ws-worker/CHANGELOG.md @@ -1,5 +1,16 @@ # ws-worker +## 0.2.7 + +### Patch Changes + +- d542aa9: worker: leave attempt channel when finished working +- Updated dependencies [793d523] +- Updated dependencies [857c42b] +- Updated dependencies [f17fb4a] + - @openfn/engine-multi@0.1.11 + - @openfn/runtime@0.1.4 + ## 0.2.6 ### Patch Changes diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json index 1bd0994e9..32c4a3aed 100644 --- a/packages/ws-worker/package.json +++ b/packages/ws-worker/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/ws-worker", - "version": "0.2.6", + "version": "0.2.7", "description": "A Websocket Worker to connect Lightning to a Runtime Engine", "main": "dist/index.js", "type": "module", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66a8db998..3c352c711 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -146,8 +146,6 @@ importers: specifier: ^5.1.6 version: 5.1.6 - integration-tests/worker/tmp/repo/exit-reason: {} - packages/cli: dependencies: '@inquirer/prompts':