From 0b510b5e6eceffe17d1d6fa19ef72db8847becb0 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 14:59:54 +0000
Subject: [PATCH 01/28] runtime: added unit tests

---
 packages/runtime/src/execute/expression.ts    |   3 +-
 .../test/execute/wrap-operation.test.ts       | 126 ++++++++++++++++++
 2 files changed, 128 insertions(+), 1 deletion(-)
 create mode 100644 packages/runtime/test/execute/wrap-operation.test.ts

diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts
index b280fe0d8..f96fd79e0 100644
--- a/packages/runtime/src/execute/expression.ts
+++ b/packages/runtime/src/execute/expression.ts
@@ -73,6 +73,7 @@ export default (
       // return the final state
       resolve(finalState);
     } catch (e: any) {
+      console.log('@@ RUNTIME:', e);
       // whatever initial state looks like now, clean it and report it back
       const finalState = prepareFinalState(opts, initialState);
       duration = Date.now() - duration;
@@ -93,7 +94,7 @@ export default (
   });
 
 // Wrap an operation with various useful stuff
-const wrapOperation = (
+export const wrapOperation = (
   fn: Operation,
   logger: Logger,
   name: string,
diff --git a/packages/runtime/test/execute/wrap-operation.test.ts b/packages/runtime/test/execute/wrap-operation.test.ts
new file mode 100644
index 000000000..8c9649a38
--- /dev/null
+++ b/packages/runtime/test/execute/wrap-operation.test.ts
@@ -0,0 +1,126 @@
+import test from 'ava';
+import { createMockLogger } from '@openfn/logger';
+
+import execute from '../../src/util/execute';
+import { wrapOperation } from '../../src/execute/expression';
+import { Operation } from '../../src/types';
+
+const logger = createMockLogger();
+
+// This function mimics the reducer created in src/execute/expression.ts
+const reducer = async (operations: Operation[], state: any) => {
+  const mapped = operations.map((op, idx) =>
+    wrapOperation(op, logger, `${idx + 1}`)
+  );
+
+  return execute(...mapped)(state);
+};
+
+test('return state', async (t) => {
+  const op = (s: any) => s;
+  const state = { x: 1 };
+
+  const result = await reducer([op], state);
+
+  t.deepEqual(result, { x: 1 });
+});
+
+test('return state async', async (t) => {
+  const op = async (s: any) => s;
+  const state = { x: 1 };
+
+  const result = await reducer([op], state);
+
+  t.deepEqual(result, { x: 1 });
+});
+
+test('call one operation', async (t) => {
+  const op = (s: any) => s + 1;
+
+  const result = await reducer([op], 1);
+
+  t.deepEqual(result, 2);
+});
+
+test('call several operations', async (t) => {
+  const op = async (s: any) => s + 1;
+
+  const result = await reducer([op, op, op], 0);
+
+  t.deepEqual(result, 3);
+});
+
+test('catch a thrown error', async (t) => {
+  const op = () => {
+    throw new Error('err');
+  };
+
+  await t.throwsAsync(() => reducer([op], {}), {
+    message: 'err',
+  });
+});
+
+test('catch a thrown error async', async (t) => {
+  const op = async () => {
+    throw new Error('err');
+  };
+
+  await t.throwsAsync(() => reducer([op], {}), {
+    message: 'err',
+  });
+});
+
+test('catch a thrown nested reference error', async (t) => {
+  const op = () => {
+    const doTheThing = () => {
+      // @ts-ignore
+      unknown.doTheThing();
+    };
+
+    doTheThing();
+  };
+
+  await t.throwsAsync(() => reducer([op], {}), {
+    name: 'ReferenceError',
+    message: 'unknown is not defined',
+  });
+});
+
+test('catch a thrown nested reference error in a promise', async (t) => {
+  const op = () =>
+    new Promise(() => {
+      const doTheThing = () => {
+        // @ts-ignore
+        unknown.doTheThing();
+      };
+
+      doTheThing();
+    });
+
+  await t.throwsAsync(() => reducer([op], {}), {
+    name: 'ReferenceError',
+    message: 'unknown is not defined',
+  });
+});
+
+test('catch an illegal function call', async (t) => {
+  const op = async (s: any) => {
+    s();
+  };
+
+  await t.throwsAsync(() => reducer([op], {}), {
+    name: 'TypeError',
+    message: 's is not a function',
+  });
+});
+
+test('catch an indirect type error', async (t) => {
+  const op = (x: any) => {
+    return async (_s: any) => x();
+  };
+
+  await t.throwsAsync(() => reducer([op('jam')], {}), {
+    name: 'TypeError',
+    message: 'x is not a function',
+  });
+});

From bbdac4a8d0cb51ccc78ba876ebdb92b6f5eb16c7 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 15:00:29 +0000
Subject: [PATCH 02/28] runtime: remove .only

---
 packages/runtime/test/runtime.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts
index 8d3047d67..76ca5cd8a 100644
--- a/packages/runtime/test/runtime.test.ts
+++ b/packages/runtime/test/runtime.test.ts
@@ -17,7 +17,7 @@ import run from '../src/runtime';
 // create large arrays or something to inflate memory usage
 // https://www.valentinog.com/blog/node-usage/
 
-test.only('run simple expression', async (t) => {
+test('run simple expression', async (t) => {
   const expression = 'export default [(s) => {s.data.done = true; return s}]';
 
   const result: any = await run(expression);

From ee10b220f0d4ece348af7b05572840968d35a7c7 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 15:01:35 +0000
Subject: [PATCH 03/28] runtime: remove comments

---
 packages/runtime/test/runtime.test.ts | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts
index 76ca5cd8a..a5ecc1127 100644
--- a/packages/runtime/test/runtime.test.ts
+++ b/packages/runtime/test/runtime.test.ts
@@ -11,12 +11,6 @@ import {
 } from '../src';
 import run from '../src/runtime';
 
-// High level examples of runtime usages
-
-// TODO create memory test
-// create large arrays or something to inflate memory usage
-// https://www.valentinog.com/blog/node-usage/
-
 test('run simple expression', async (t) => {
   const expression = 'export default [(s) => {s.data.done = true; return s}]';
 

From 2155cd6c5789e01ee2f61d2d3cbeb57cb8c85ccc Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 15:19:31 +0000
Subject: [PATCH 04/28] runtime: more tests

---
 .../runtime/test/__modules__/test/index.js    | 11 +++++
 packages/runtime/test/runtime.test.ts         | 44 +++++++++++++++++++
 2 files changed, 55 insertions(+)

diff --git a/packages/runtime/test/__modules__/test/index.js b/packages/runtime/test/__modules__/test/index.js
index 15b04e0b6..247835937 100644
--- a/packages/runtime/test/__modules__/test/index.js
+++ b/packages/runtime/test/__modules__/test/index.js
@@ -9,3 +9,14 @@ export const err = () => {
 export const err2 = () => {
   throw 'adaptor err';
 };
+
+// trying to repro https://github.com/OpenFn/kit/issues/520
+export function call(fn) {
+  return (state) => {
+    try {
+      return { data: fn(state) };
+    } catch (e) {
+      throw e;
+    }
+  };
+}
diff --git a/packages/runtime/test/runtime.test.ts b/packages/runtime/test/runtime.test.ts
index a5ecc1127..103eb4668 100644
--- a/packages/runtime/test/runtime.test.ts
+++ b/packages/runtime/test/runtime.test.ts
@@ -516,3 +516,47 @@ test("injected globals can't override special functions", async (t) => {
   const result: any = await run(expression, {}, { globals });
   t.falsy(result.errors);
 });
+
+test('run from an adaptor', async (t) => {
+  const expression = `
+    import { call } from 'x';
+    export default [call(() => 22)];
+  `;
+
+  const result: any = await run(
+    expression,
+    {},
+    {
+      linker: {
+        modules: {
+          x: { path: path.resolve('test/__modules__/test') },
+        },
+      },
+    }
+  );
+
+  t.deepEqual(result, { data: 22 });
+});
+
+// https://github.com/OpenFn/kit/issues/520
+test('run from an adaptor with error', async (t) => {
+  const expression = `
+    import { call } from 'x';
+    export default [call("22")];
+  `;
+
+  const result: any = await run(
+    expression,
+    {},
+    {
+      linker: {
+        modules: {
+          x: { path: path.resolve('test/__modules__/test') },
+        },
+      },
+    }
+  );
+
+  // This should safely return with an error
+  t.truthy(result.errors['job-1']);
+});

From ffa06e784b876cd5f869bfdb796f91c330742deb Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 16:21:24 +0000
Subject: [PATCH 05/28] runtime: remove logging

---
 packages/runtime/src/execute/expression.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/runtime/src/execute/expression.ts b/packages/runtime/src/execute/expression.ts
index f96fd79e0..9e2effc72 100644
--- a/packages/runtime/src/execute/expression.ts
+++ b/packages/runtime/src/execute/expression.ts
@@ -73,7 +73,6 @@ export default (
       // return the final state
       resolve(finalState);
     } catch (e: any) {
-      console.log('@@ RUNTIME:', e);
       // whatever initial state looks like now, clean it and report it back
       const finalState = prepareFinalState(opts, initialState);
       duration = Date.now() - duration;

From 1374117f9a46380dc96802d0d330cb6050e079ec Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 16:34:45 +0000
Subject: [PATCH 06/28] engine: add repo to test folder

---
 .../test/__repo__/node_modules/helper_1.0.0/index.cjs     | 6 ++++++
 .../test/__repo__/node_modules/helper_1.0.0/package.json  | 7 +++++++
 packages/engine-multi/test/__repo__/package.json          | 8 ++++++++
 3 files changed, 21 insertions(+)
 create mode 100644 packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
 create mode 100644 packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json
 create mode 100644 packages/engine-multi/test/__repo__/package.json

diff --git a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
new file mode 100644
index 000000000..5c6531b7b
--- /dev/null
+++ b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
@@ -0,0 +1,6 @@
+module.exports =  {
+  exit: function() {
+    console.log('exiting process')
+    process.exit(1)
+  }
+};
diff --git a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json
new file mode 100644
index 000000000..429f14f13
--- /dev/null
+++ b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "helper",
+  "version": "1.0.0",
+  "type": "module",
+  "main": "index.cjs",
+  "private": true
+}
diff --git a/packages/engine-multi/test/__repo__/package.json b/packages/engine-multi/test/__repo__/package.json
new file mode 100644
index 000000000..679a959d6
--- /dev/null
+++ b/packages/engine-multi/test/__repo__/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "test-repo",
+  "private": true,
+  "version": "1.0.0",
+  "dependencies": {
+    "helper_1.0.0": "@npm:helper@1.0.0"
+  }
+}

From 5d1845de0322312793ae151c90802d5127c4332e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 17:35:21 +0000
Subject: [PATCH 07/28] engine: catch process.exit from inside the thread

---
 packages/engine-multi/src/api/call-worker.ts  |  4 +++
 packages/engine-multi/src/api/execute.ts      | 16 +++++------
 packages/engine-multi/src/errors.ts           | 14 ++++++++++
 .../engine-multi/src/worker/worker-helper.ts  |  3 +-
 .../node_modules/helper_1.0.0/index.cjs       |  7 +++--
 packages/engine-multi/test/errors.test.ts     | 28 +++++++++++++++++--
 6 files changed, 57 insertions(+), 15 deletions(-)

diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts
index 257175705..e08b2abe4 100644
--- a/packages/engine-multi/src/api/call-worker.ts
+++ b/packages/engine-multi/src/api/call-worker.ts
@@ -96,5 +96,9 @@ export function createWorkers(workerPath: string, options: WorkerOptions) {
         maxOldGenerationSizeMb: memoryLimitMb,
       },
     },
+    onTerminateWorker: (evt) => {
+      console.log(' **** worker terminated ');
+      console.log(evt);
+    },
   });
 }
diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts
index 1f19b5abe..675d3644e 100644
--- a/packages/engine-multi/src/api/execute.ts
+++ b/packages/engine-multi/src/api/execute.ts
@@ -16,7 +16,7 @@ import {
   jobError,
 } from './lifecycle';
 import preloadCredentials from './preload-credentials';
-import { ExecutionError, OOMError, TimeoutError } from '../errors';
+import { ExecutionError, ExitError, OOMError, TimeoutError } from '../errors';
 
 const execute = async (context: ExecutionContext) => {
   const { state, callWorker, logger, options } = context;
@@ -84,20 +84,18 @@ const execute = async (context: ExecutionContext) => {
       events,
       options.timeout
     ).catch((e: any) => {
-      // An error here is basically a crash state
-      if (e.code === 'ERR_WORKER_OUT_OF_MEMORY') {
+      // Catch process.exit from inside the thread
+      // This approach is not pretty - we are banking on replacing workerpool soon
+      if (e.message.match(/^Workerpool Worker terminated Unexpectedly/)) {
+        const exitCode = e.message.match(/exitCode: `(\d+)`/);
+        e = new ExitError(parseInt(exitCode[1]));
+      } else if (e.code === 'ERR_WORKER_OUT_OF_MEMORY') {
         e = new OOMError();
       } else if (e instanceof WorkerPoolPromise.TimeoutError) {
         // Map the workerpool error to our own
         e = new TimeoutError(options.timeout!);
       }
 
-      // TODO: map anything else to an executionError
-
-      // TODO what information can I usefully provide here?
-      // DO I know which job I'm on?
-      // DO I know the thread id?
-      // Do I know where the error came from?
       error(context, { workflowId: state.plan.id, error: e });
       logger.error(e);
     });
diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts
index 46633a55a..fa2156b83 100644
--- a/packages/engine-multi/src/errors.ts
+++ b/packages/engine-multi/src/errors.ts
@@ -82,4 +82,18 @@ export class OOMError extends EngineError {
   }
 }
 
+export class ExitError extends EngineError {
+  severity = 'kill';
+  type = 'ExitError';
+  name = 'ExitError';
+  code;
+  message;
+
+  constructor(code: number) {
+    super();
+    this.code = code;
+    this.message = `Process exited with code: ${code}`;
+  }
+}
+
 // CredentialsError (exception)
diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts
index 9562b0654..99f50574b 100644
--- a/packages/engine-multi/src/worker/worker-helper.ts
+++ b/packages/engine-multi/src/worker/worker-helper.ts
@@ -6,7 +6,7 @@ import { threadId } from 'node:worker_threads';
 import createLogger, { SanitizePolicies } from '@openfn/logger';
 
 import * as workerEvents from './events';
-import { ExecutionError } from '../errors';
+import { ExecutionError, ExitError } from '../errors';
 
 export const createLoggers = (
   workflowId: string,
@@ -90,7 +90,6 @@ async function helper(workflowId: string, execute: () => Promise<any>) {
     const e = new ExecutionError(err);
     e.severity = 'crash'; // Downgrade this to a crash becuase it's likely not our fault
     handleError(e);
-    process.exit(1);
   });
 
   try {
diff --git a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
index 5c6531b7b..a12a1a2b0 100644
--- a/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
+++ b/packages/engine-multi/test/__repo__/node_modules/helper_1.0.0/index.cjs
@@ -1,6 +1,9 @@
 module.exports =  {
   exit: function() {
-    console.log('exiting process')
-    process.exit(1)
+    return function (state) {
+      console.log('exiting process')
+      process.exit(42)
+      return state;
+    }
   }
 };
diff --git a/packages/engine-multi/test/errors.test.ts b/packages/engine-multi/test/errors.test.ts
index ea7295cb7..bc0af8813 100644
--- a/packages/engine-multi/test/errors.test.ts
+++ b/packages/engine-multi/test/errors.test.ts
@@ -1,7 +1,9 @@
 import test from 'ava';
+import path from 'node:path';
+
 import createEngine, { EngineOptions } from '../src/engine';
 import { createMockLogger } from '@openfn/logger';
-import { WORKFLOW_ERROR } from '../src/events';
+import { WORKFLOW_COMPLETE, WORKFLOW_ERROR } from '../src/events';
 
 let engine;
 
@@ -10,7 +12,7 @@ test.before(async () => {
 
   const options: EngineOptions = {
     logger,
-    repoDir: '.',
+    repoDir: path.resolve('./test/__repo__'),
     autoinstall: {
       // disable autoinstall
       handleIsInstalled: async () => true,
@@ -119,3 +121,25 @@ test.serial('execution error from async code', (t) => {
     });
   });
 });
+
+// This passes standaloen but fails alongside others? Curious
+test.serial.skip('process.exit', (t) => {
+  return new Promise((done) => {
+    const plan = {
+      id: 'a',
+      jobs: [
+        {
+          adaptor: 'helper@1.0.0',
+          expression: `export default [exit()]`,
+        },
+      ],
+    };
+
+    engine.execute(plan).on(WORKFLOW_ERROR, (evt) => {
+      t.is(evt.type, 'ExitError');
+      t.is(evt.severity, 'kill');
+      t.is(evt.message, 'Process exited with code: 42');
+      done();
+    });
+  });
+});

From 9a86e2eade06acddb017b7074abf693ba4502037 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 29 Nov 2023 17:37:36 +0000
Subject: [PATCH 08/28] engine: remove unused code

---
 packages/engine-multi/src/api/call-worker.ts | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/packages/engine-multi/src/api/call-worker.ts b/packages/engine-multi/src/api/call-worker.ts
index e08b2abe4..257175705 100644
--- a/packages/engine-multi/src/api/call-worker.ts
+++ b/packages/engine-multi/src/api/call-worker.ts
@@ -96,9 +96,5 @@ export function createWorkers(workerPath: string, options: WorkerOptions) {
         maxOldGenerationSizeMb: memoryLimitMb,
       },
     },
-    onTerminateWorker: (evt) => {
-      console.log(' **** worker terminated ');
-      console.log(evt);
-    },
   });
 }

From 086680c054e944d70a130fe531db3ebf3d1545a4 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 11:23:16 +0000
Subject: [PATCH 09/28] typing

---
 packages/engine-multi/src/worker/worker-helper.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts
index 99f50574b..4ca1a3260 100644
--- a/packages/engine-multi/src/worker/worker-helper.ts
+++ b/packages/engine-multi/src/worker/worker-helper.ts
@@ -6,7 +6,7 @@ import { threadId } from 'node:worker_threads';
 import createLogger, { SanitizePolicies } from '@openfn/logger';
 
 import * as workerEvents from './events';
-import { ExecutionError, ExitError } from '../errors';
+import { ExecutionError } from '../errors';
 
 export const createLoggers = (
   workflowId: string,

From e4f22a6ff4083f70350ebd48f023d6c75065d4c0 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 11:37:45 +0000
Subject: [PATCH 10/28] engine: tweak error messaging

---
 packages/engine-multi/src/api/execute.ts | 2 +-
 packages/engine-multi/src/errors.ts      | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts
index 675d3644e..cf7000df1 100644
--- a/packages/engine-multi/src/api/execute.ts
+++ b/packages/engine-multi/src/api/execute.ts
@@ -97,7 +97,7 @@ const execute = async (context: ExecutionContext) => {
       }
 
       error(context, { workflowId: state.plan.id, error: e });
-      logger.error(e);
+      logger.error(`Critical error thrown by ${state.plan.id}`, e);
     });
   } catch (e: any) {
     if (!e.severity) {
diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts
index fa2156b83..a65b8d78e 100644
--- a/packages/engine-multi/src/errors.ts
+++ b/packages/engine-multi/src/errors.ts
@@ -93,6 +93,9 @@ export class ExitError extends EngineError {
     super();
     this.code = code;
     this.message = `Process exited with code: ${code}`;
+    // Remove the stack trace
+    // It contains no useful information
+    this.stack = '';
   }
 }
 

From 05c2d83d7c0e7d6a6941a836e0da0b8f601afb48 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 12:36:03 +0000
Subject: [PATCH 11/28] engine: exit the worker proces son error, just in case

There can be problems where async code is still running - to be absolutely sure, we kill the process on error
---
 packages/engine-multi/src/api/execute.ts          |  9 +++++++--
 packages/engine-multi/src/worker/worker-helper.ts | 13 +++++++++----
 packages/engine-multi/test/errors.test.ts         | 15 ++++++++-------
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts
index cf7000df1..2eb387ebe 100644
--- a/packages/engine-multi/src/api/execute.ts
+++ b/packages/engine-multi/src/api/execute.ts
@@ -87,8 +87,13 @@ const execute = async (context: ExecutionContext) => {
       // Catch process.exit from inside the thread
       // This approach is not pretty - we are banking on replacing workerpool soon
       if (e.message.match(/^Workerpool Worker terminated Unexpectedly/)) {
-        const exitCode = e.message.match(/exitCode: `(\d+)`/);
-        e = new ExitError(parseInt(exitCode[1]));
+        const [_match, exitCode] = e.message.match(/exitCode: `(\d+)`/);
+        if (exitCode === '111111') {
+          // This means a controlled exit from inside the worker
+          // The error has already been reported and we should do nothing
+          return;
+        }
+        e = new ExitError(parseInt(exitCode));
       } else if (e.code === 'ERR_WORKER_OUT_OF_MEMORY') {
         e = new OOMError();
       } else if (e instanceof WorkerPoolPromise.TimeoutError) {
diff --git a/packages/engine-multi/src/worker/worker-helper.ts b/packages/engine-multi/src/worker/worker-helper.ts
index 4ca1a3260..66c64d15f 100644
--- a/packages/engine-multi/src/worker/worker-helper.ts
+++ b/packages/engine-multi/src/worker/worker-helper.ts
@@ -88,14 +88,19 @@ async function helper(workflowId: string, execute: () => Promise<any>) {
     // For now, we'll write this off as a crash-level generic execution error
     // TODO did this come from job or adaptor code?
     const e = new ExecutionError(err);
-    e.severity = 'crash'; // Downgrade this to a crash becuase it's likely not our fault
+    e.severity = 'crash'; // Downgrade this to a crash because it's likely not our fault
     handleError(e);
+
+    // Close down the process justto be 100% sure that all async code stops
+    // This is in a timeout to give the emitted message time to escape
+    // There is a TINY WINDOW in which async code can still run and affect the next attempt
+    // This should all go away when we replace workerpool
+    setTimeout(() => {
+      process.exit(111111);
+    }, 2);
   });
 
   try {
-    // Note that the worker thread may fire logs after completion
-    // I think this is fine, it's just a log stream thing
-    // But the output is very confusing!
     const result = await execute();
     publish(workflowId, workerEvents.WORKFLOW_COMPLETE, { state: result });
 
diff --git a/packages/engine-multi/test/errors.test.ts b/packages/engine-multi/test/errors.test.ts
index bc0af8813..88914cc3b 100644
--- a/packages/engine-multi/test/errors.test.ts
+++ b/packages/engine-multi/test/errors.test.ts
@@ -104,11 +104,12 @@ test.serial('execution error from async code', (t) => {
       id: 'a',
       jobs: [
         {
+          // this error will throw within the promise, and so before the job completes
+          // But REALLY naughty code could throw after the job has finished
+          // In which case it'll be ignored
+          // Also note that the wrapping promise will never resolve
           expression: `export default [(s) => new Promise((r) => {
-            // this error will throw within the promise, and so before the job completes
-            // But REALLY naughty code could throw after the job has finished
-            // In which case it'll be ignored
-            setTimeout(() => { throw new Error(\"e\");r () }, 1)
+            setTimeout(() => { throw new Error(\"e1324\"); r() }, 10)
             })]`,
         },
       ],
@@ -117,16 +118,16 @@ test.serial('execution error from async code', (t) => {
     engine.execute(plan).on(WORKFLOW_ERROR, (evt) => {
       t.is(evt.type, 'ExecutionError');
       t.is(evt.severity, 'crash');
+
       done();
     });
   });
 });
 
-// This passes standaloen but fails alongside others? Curious
-test.serial.skip('process.exit', (t) => {
+test.serial('emit a kill error on process.exit()', (t) => {
   return new Promise((done) => {
     const plan = {
-      id: 'a',
+      id: 'z',
       jobs: [
         {
           adaptor: 'helper@1.0.0',

From 7235bf5eed22ce8990a8296943df00f65e9fdf15 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 12:58:34 +0000
Subject: [PATCH 12/28] changeset

---
 .changeset/violet-ligers-walk.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/violet-ligers-walk.md

diff --git a/.changeset/violet-ligers-walk.md b/.changeset/violet-ligers-walk.md
new file mode 100644
index 000000000..623d0e174
--- /dev/null
+++ b/.changeset/violet-ligers-walk.md
@@ -0,0 +1,5 @@
+---
+'@openfn/engine-multi': patch
+---
+
+Throw a better error on process.exit

From 6c3e9e42356a887df6d807e1a68e9f81554262c8 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 14:25:04 +0000
Subject: [PATCH 13/28] worker: ensure capacity is also set on the engine

---
 .changeset/moody-mice-swim.md   |  5 +++++
 packages/ws-worker/src/start.ts | 14 ++++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)
 create mode 100644 .changeset/moody-mice-swim.md

diff --git a/.changeset/moody-mice-swim.md b/.changeset/moody-mice-swim.md
new file mode 100644
index 000000000..cb8d605ee
--- /dev/null
+++ b/.changeset/moody-mice-swim.md
@@ -0,0 +1,5 @@
+---
+'@openfn/ws-worker': patch
+---
+
+Ensure capacity is also set on the engine
diff --git a/packages/ws-worker/src/start.ts b/packages/ws-worker/src/start.ts
index e23a00b26..5c22d612e 100644
--- a/packages/ws-worker/src/start.ts
+++ b/packages/ws-worker/src/start.ts
@@ -120,10 +120,12 @@ if (args.mock) {
     engineReady(engine);
   });
 } else {
-  createRTE({ repoDir: args.repoDir, memoryLimitMb: args.runMemory }).then(
-    (engine) => {
-      logger.debug('engine created');
-      engineReady(engine);
-    }
-  );
+  createRTE({
+    repoDir: args.repoDir,
+    memoryLimitMb: args.runMemory,
+    maxWorkers: args.capacity,
+  }).then((engine) => {
+    logger.debug('engine created');
+    engineReady(engine);
+  });
 }

From 101f38a06ec1a2dd94174fd034cb24aca1d01935 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 09:58:30 +0000
Subject: [PATCH 14/28] worker: downgrade exiterror to a crash

---
 packages/engine-multi/src/errors.ts       | 2 +-
 packages/engine-multi/test/errors.test.ts | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts
index a65b8d78e..f26c0613c 100644
--- a/packages/engine-multi/src/errors.ts
+++ b/packages/engine-multi/src/errors.ts
@@ -83,7 +83,7 @@ export class OOMError extends EngineError {
 }
 
 export class ExitError extends EngineError {
-  severity = 'kill';
+  severity = 'crash';
   type = 'ExitError';
   name = 'ExitError';
   code;
diff --git a/packages/engine-multi/test/errors.test.ts b/packages/engine-multi/test/errors.test.ts
index 88914cc3b..ad6f4f9e3 100644
--- a/packages/engine-multi/test/errors.test.ts
+++ b/packages/engine-multi/test/errors.test.ts
@@ -124,7 +124,7 @@ test.serial('execution error from async code', (t) => {
   });
 });
 
-test.serial('emit a kill error on process.exit()', (t) => {
+test.serial('emit a crash error on process.exit()', (t) => {
   return new Promise((done) => {
     const plan = {
       id: 'z',
@@ -138,7 +138,7 @@ test.serial('emit a kill error on process.exit()', (t) => {
 
     engine.execute(plan).on(WORKFLOW_ERROR, (evt) => {
       t.is(evt.type, 'ExitError');
-      t.is(evt.severity, 'kill');
+      t.is(evt.severity, 'crash');
       t.is(evt.message, 'Process exited with code: 42');
       done();
     });

From 78755427bb503ab433e55333c8a2e888b16edb6e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 09:58:39 +0000
Subject: [PATCH 15/28] test: trigger a process.exit

---
 .../worker/test/exit-reasons.test.ts          | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/integration-tests/worker/test/exit-reasons.test.ts b/integration-tests/worker/test/exit-reasons.test.ts
index fb3218951..f8e981276 100644
--- a/integration-tests/worker/test/exit-reasons.test.ts
+++ b/integration-tests/worker/test/exit-reasons.test.ts
@@ -100,3 +100,23 @@ test('kill: oom', async (t) => {
   t.is(error_type, 'OOMError');
   t.is(error_message, 'Run exceeded maximum memory usage');
 });
+
+test('crash: process.exit() triggered by postgres', async (t) => {
+  const attempt = {
+    id: crypto.randomUUID(),
+    jobs: [
+      {
+        adaptor: '@openfn/language-postgresql@4.1.8', // version number is important
+        body: "sql('select * from food_hygiene_interview');",
+      },
+    ],
+  };
+
+  const result = await run(attempt);
+
+  const { reason, error_type, error_message } = result;
+
+  t.is(reason, 'crash');
+  t.is(error_type, 'ExitError');
+  t.regex(error_message, /Process exited with code: 1/i);
+});

From 86dfa41509743766fff0818a512de9a0844ccd65 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Tue, 28 Nov 2023 19:02:22 +0000
Subject: [PATCH 16/28] worker: trigger job error on workflow error to close
 out the job

---
 packages/ws-worker/src/api/execute.ts       | 19 +++++----
 packages/ws-worker/test/api/execute.test.ts | 47 ++++++++++++++++++++-
 2 files changed, 57 insertions(+), 9 deletions(-)

diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts
index 0df599976..06a9b201f 100644
--- a/packages/ws-worker/src/api/execute.ts
+++ b/packages/ws-worker/src/api/execute.ts
@@ -188,7 +188,7 @@ export function onJobError(context: Context, event: any) {
   // because it'll count it as a crash
   // This isn't very good: maybe we shouldn't trigger an error
   // at all for a fail state?
-  const { state, error, jobId } = event;
+  const { state = {}, error, jobId } = event;
   // This test is horrible too
   if (state.errors?.[jobId]?.message === error.message) {
     onJobComplete(context, event);
@@ -277,19 +277,21 @@ export async function onWorkflowComplete(
   onFinish({ reason, state: result });
 }
 
-// On error, for now, we just post to workflow complete
-// No unit tests on this (not least because I think it'll change soon)
-// NB this is a crash state!
 export async function onWorkflowError(
-  { state, channel, logger, onFinish }: Context,
+  context: Context,
   event: WorkflowErrorPayload
 ) {
-  // Should we not just report this reason?
-  // Nothing more severe can have happened downstream, right?
-  // const reason = calculateAttemptExitReason(state);
+  const { state, channel, logger, onFinish } = context;
+
   try {
     // Ok, let's try that, let's just generate a reason from the event
     const reason = calculateJobExitReason('', { data: {} }, event);
+
+    // If there's a job still running, make sure it gets marked complete
+    if (state.activeJob) {
+      await onJobError(context, event);
+    }
+
     await sendEvent<AttemptCompletePayload>(channel, ATTEMPT_COMPLETE, {
       final_dataclip_id: state.lastDataclipId!,
       ...reason,
@@ -297,6 +299,7 @@ export async function onWorkflowError(
 
     onFinish({ reason });
   } catch (e: any) {
+    console.log(e);
     logger.error('ERROR in workflow-error handler:', e.message);
     logger.error(e);
   }
diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts
index 2b73a61a4..6f53537ad 100644
--- a/packages/ws-worker/test/api/execute.test.ts
+++ b/packages/ws-worker/test/api/execute.test.ts
@@ -17,6 +17,7 @@ import {
   execute,
   onWorkflowStart,
   onWorkflowComplete,
+  onWorkflowError,
   loadDataclip,
   loadCredential,
   sendEvent,
@@ -27,6 +28,11 @@ import { stringify, createAttemptState } from '../../src/util';
 
 import type { ExecutionPlan } from '@openfn/runtime';
 import type { AttemptState } from '../../src/types';
+import {
+  JOB_COMPLETE,
+  JOB_ERROR,
+  WORKFLOW_COMPLETE,
+} from '@openfn/engine-multi';
 
 const enc = new TextEncoder();
 
@@ -220,7 +226,7 @@ test('jobComplete should generate an exit reason: success', async (t) => {
   t.is(event.error_message, null);
 });
 
-test.only('jobComplete should send a run:complete event', async (t) => {
+test('jobComplete should send a run:complete event', async (t) => {
   const plan = { id: 'attempt-1' };
   const jobId = 'job-1';
   const result = { x: 10 };
@@ -382,6 +388,45 @@ test('workflowComplete should call onFinish with final dataclip', async (t) => {
   await onWorkflowComplete(context, event);
 });
 
+test.only('workflowError should trigger runComplete with a reason', async (t) => {
+  const plan = { id: 'attempt-1' };
+  const jobId = 'job-1';
+
+  const state = {
+    reasons: {},
+    dataclips: {},
+    lastDataclipId: 'x',
+    activeJob: jobId,
+    activeRun: 'b',
+    errors: {},
+  };
+
+  const channel = mockChannel({
+    [RUN_COMPLETE]: (evt) => {
+      t.is(evt.reason, 'crash');
+      t.pass('called job complete');
+      return true;
+    },
+    [ATTEMPT_COMPLETE]: () => true,
+  });
+
+  const event = {
+    error: {
+      severity: 'crash',
+      type: 'Err',
+      message: 'it crashed',
+    },
+    state: {},
+  };
+
+  const context = { channel, state, onFinish: () => {} };
+
+  await onWorkflowError(context, event);
+});
+
+test.todo('send the reason to onFinish');
+test.todo("don't call complete if the job is not active ");
+
 // TODO what if an error?
 test('loadDataclip should fetch a dataclip', async (t) => {
   const channel = mockChannel({

From 740fb1262848bae849c1c97503328ac38393cc4e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Tue, 28 Nov 2023 19:06:22 +0000
Subject: [PATCH 17/28] more tests

---
 packages/ws-worker/src/api/execute.ts       |  1 -
 packages/ws-worker/test/api/execute.test.ts | 80 +++++++++++++++++++--
 2 files changed, 75 insertions(+), 6 deletions(-)

diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts
index 06a9b201f..ca94f6e0c 100644
--- a/packages/ws-worker/src/api/execute.ts
+++ b/packages/ws-worker/src/api/execute.ts
@@ -299,7 +299,6 @@ export async function onWorkflowError(
 
     onFinish({ reason });
   } catch (e: any) {
-    console.log(e);
     logger.error('ERROR in workflow-error handler:', e.message);
     logger.error(e);
   }
diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts
index 6f53537ad..3e57990b1 100644
--- a/packages/ws-worker/test/api/execute.test.ts
+++ b/packages/ws-worker/test/api/execute.test.ts
@@ -388,8 +388,7 @@ test('workflowComplete should call onFinish with final dataclip', async (t) => {
   await onWorkflowComplete(context, event);
 });
 
-test.only('workflowError should trigger runComplete with a reason', async (t) => {
-  const plan = { id: 'attempt-1' };
+test('workflowError should trigger runComplete with a reason', async (t) => {
   const jobId = 'job-1';
 
   const state = {
@@ -404,7 +403,6 @@ test.only('workflowError should trigger runComplete with a reason', async (t) =>
   const channel = mockChannel({
     [RUN_COMPLETE]: (evt) => {
       t.is(evt.reason, 'crash');
-      t.pass('called job complete');
       return true;
     },
     [ATTEMPT_COMPLETE]: () => true,
@@ -424,8 +422,80 @@ test.only('workflowError should trigger runComplete with a reason', async (t) =>
   await onWorkflowError(context, event);
 });
 
-test.todo('send the reason to onFinish');
-test.todo("don't call complete if the job is not active ");
+test('workflow error should send reason to onFinish', async (t) => {
+  const jobId = 'job-1';
+
+  const state = {
+    reasons: {},
+    dataclips: {},
+    lastDataclipId: 'x',
+    activeJob: jobId,
+    activeRun: 'b',
+    errors: {},
+  };
+
+  const channel = mockChannel({
+    [RUN_COMPLETE]: (evt) => true,
+    [ATTEMPT_COMPLETE]: () => true,
+  });
+
+  const event = {
+    error: {
+      severity: 'crash',
+      type: 'Err',
+      message: 'it crashed',
+    },
+    state: {},
+  };
+
+  const context = {
+    channel,
+    state,
+    onFinish: (evt) => {
+      t.is(evt.reason.reason, 'crash');
+    },
+  };
+
+  await onWorkflowError(context, event);
+});
+
+test('workflowError should not call job complete if the job is not active', async (t) => {
+  const state = {
+    reasons: {},
+    dataclips: {},
+    lastDataclipId: 'x',
+    activeJob: undefined,
+    activeRun: undefined,
+    errors: {},
+  };
+
+  const channel = mockChannel({
+    [RUN_COMPLETE]: (evt) => {
+      t.fail('should not call!');
+      return true;
+    },
+    [ATTEMPT_COMPLETE]: () => true,
+  });
+
+  const event = {
+    error: {
+      severity: 'crash',
+      type: 'Err',
+      message: 'it crashed',
+    },
+    state: {},
+  };
+
+  const context = {
+    channel,
+    state,
+    onFinish: () => {
+      t.pass();
+    },
+  };
+
+  await onWorkflowError(context, event);
+});
 
 // TODO what if an error?
 test('loadDataclip should fetch a dataclip', async (t) => {

From 213181968e545ed07056c3238045a190072ade2e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 15:11:39 +0000
Subject: [PATCH 18/28] worker: test for timeout in engine mock

---
 packages/ws-worker/src/mock/runtime-engine.ts |  6 ++--
 packages/ws-worker/test/lightning.test.ts     | 32 +++++++++++++++++--
 .../test/mock/runtime-engine.test.ts          | 17 +++++++++-
 3 files changed, 49 insertions(+), 6 deletions(-)

diff --git a/packages/ws-worker/src/mock/runtime-engine.ts b/packages/ws-worker/src/mock/runtime-engine.ts
index ebb87a5c3..0108ca2af 100644
--- a/packages/ws-worker/src/mock/runtime-engine.ts
+++ b/packages/ws-worker/src/mock/runtime-engine.ts
@@ -132,6 +132,7 @@ async function createMock() {
 
       try {
         await run(xplan, undefined, opts as any);
+        dispatch('workflow-complete', { workflowId: id, threadId: threadId });
       } catch (e: any) {
         dispatch('workflow-error', {
           threadId: threadId,
@@ -139,10 +140,9 @@ async function createMock() {
           type: e.name,
           message: e.message,
         });
+      } finally {
+        delete activeWorkflows[id!];
       }
-
-      delete activeWorkflows[id!];
-      dispatch('workflow-complete', { workflowId: id, threadId: threadId });
     }, 1);
 
     // Technically the engine should return an event emitter
diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts
index a16f66d33..8980e9d07 100644
--- a/packages/ws-worker/test/lightning.test.ts
+++ b/packages/ws-worker/test/lightning.test.ts
@@ -264,14 +264,13 @@ test.serial(`events: lightning should receive a ${e.RUN_START} event`, (t) => {
   });
 });
 
-test.serial.only(
+test.serial(
   `events: lightning should receive a ${e.RUN_COMPLETE} event`,
   (t) => {
     return new Promise((done) => {
       const attempt = getAttempt();
 
       lng.onSocketEvent(e.RUN_COMPLETE, attempt.id, ({ payload }) => {
-        console.log(payload);
         t.is(payload.job_id, 'j');
         t.truthy(payload.run_id);
         t.truthy(payload.output_dataclip);
@@ -291,6 +290,35 @@ test.serial.only(
   }
 );
 
+test.serial.only(
+  `events: lightning should receive a ${e.RUN_COMPLETE} event even if the attempt fails`,
+  (t) => {
+    return new Promise((done) => {
+      // This attempt should timeout
+      const attempt = getAttempt({ options: { timeout: 100 } }, [
+        {
+          id: 'z',
+          adaptor: '@openfn/language-common@1.0.0',
+          body: 'wait(1000)',
+        },
+      ]);
+
+      lng.onSocketEvent(e.RUN_COMPLETE, attempt.id, ({ payload }) => {
+        console.log(payload);
+        t.not(payload.reason, 'success');
+        t.pass('called run complete');
+      });
+
+      lng.onSocketEvent(e.ATTEMPT_COMPLETE, attempt.id, ({ payload }) => {
+        t.not(payload.reason, 'success');
+        done();
+      });
+
+      lng.enqueueAttempt(attempt);
+    });
+  }
+);
+
 test.serial(
   `events: lightning should receive a ${e.ATTEMPT_LOG} event`,
   (t) => {
diff --git a/packages/ws-worker/test/mock/runtime-engine.test.ts b/packages/ws-worker/test/mock/runtime-engine.test.ts
index 13c3c3055..65ed5d19d 100644
--- a/packages/ws-worker/test/mock/runtime-engine.test.ts
+++ b/packages/ws-worker/test/mock/runtime-engine.test.ts
@@ -6,7 +6,7 @@ import create, {
   WorkflowStartEvent,
 } from '../../src/mock/runtime-engine';
 import type { ExecutionPlan } from '@openfn/runtime';
-import { waitForEvent, clone } from '../util'; // ???
+import { waitForEvent, clone } from '../util';
 
 const sampleWorkflow = {
   id: 'w1',
@@ -230,3 +230,18 @@ test('do nothing for a job if no expression and adaptor (trigger node)', async (
 
   t.false(didCallEvent);
 });
+
+test('timeout', async (t) => {
+  const wf = clone(sampleWorkflow);
+  wf.jobs[0].expression = 'wait(1000)';
+  // wf.options = { timeout: 10 };
+
+  // @ts-ignore
+  engine.execute(wf, { timeout: 10 });
+
+  const evt = await waitForEvent<WorkflowCompleteEvent>(
+    engine,
+    'workflow-error'
+  );
+  t.is(evt.type, 'TimeoutError');
+});

From 648b154939e38da190f62b0fcb2fd00c8e36f58b Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 17:33:19 +0000
Subject: [PATCH 19/28] tests: adding new tests

---
 .../worker/test/exit-reasons.test.ts          |  2 +-
 .../worker/test/integration.test.ts           | 64 +++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/integration-tests/worker/test/exit-reasons.test.ts b/integration-tests/worker/test/exit-reasons.test.ts
index f8e981276..89712326c 100644
--- a/integration-tests/worker/test/exit-reasons.test.ts
+++ b/integration-tests/worker/test/exit-reasons.test.ts
@@ -77,7 +77,7 @@ test('exception: autoinstall error', async (t) => {
   );
 });
 
-test('kill: oom', async (t) => {
+test.only('kill: oom', async (t) => {
   const attempt = {
     id: crypto.randomUUID(),
     jobs: [
diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts
index 38b2d0c08..c4e202555 100644
--- a/integration-tests/worker/test/integration.test.ts
+++ b/integration-tests/worker/test/integration.test.ts
@@ -327,6 +327,70 @@ test('blacklist a non-openfn adaptor', (t) => {
   });
 });
 
+// TODO this fails, I think this is a wider timeout problem but I don't really want to get into it now
+test.skip('a timeout error should still call run-complete', (t) => {
+  return new Promise(async (done) => {
+    const attempt = {
+      id: crypto.randomUUID(),
+      jobs: [
+        {
+          adaptor: '@openfn/language-common@latest', // version lock to something stable?
+          body: 'fn((s) => new Promise((resolve) => setTimeout(() => resolve(s), 1000)))',
+        },
+      ],
+      options: {
+        // Including the timeout here stops the attempt returning at all
+        timeout: 100,
+      },
+    };
+
+    lightning.once('run:complete', (event) => {
+      console.log(event.payload);
+      t.is(event.payload.reason, 'kill');
+    });
+
+    lightning.once('attempt:complete', (event) => {
+      console.log(event.payload);
+      done();
+    });
+
+    lightning.enqueueAttempt(attempt);
+  });
+});
+
+test.only('an OOM error should still call run-complete', (t) => {
+  return new Promise(async (done) => {
+    const attempt = {
+      id: crypto.randomUUID(),
+      jobs: [
+        {
+          adaptor: '@openfn/language-common@latest', // version lock to something stable?
+          body: `
+          fn((s) => {
+              s.data = [];
+              while(true) {
+                s.data.push(new Array(1e5).fill("xyz"))
+              }
+              return s;
+          })`,
+        },
+      ],
+    };
+
+    // lightning.once('run:complete', (event) => {
+    //   console.log(event.payload);
+    //   t.is(event.payload.reason, 'kill');
+    // });
+
+    lightning.once('attempt:complete', (event) => {
+      console.log(event.payload);
+      done();
+    });
+
+    lightning.enqueueAttempt(attempt);
+  });
+});
+
 // test('run a job with complex behaviours (initial state, branching)', (t) => {
 //   const attempt = {
 //     id: 'a1',

From d5447bd59ab390c0cb4975955270feef6a0845e6 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 18:27:43 +0000
Subject: [PATCH 20/28] tests: add new tests for run-complete

---
 .../worker/test/exit-reasons.test.ts              |  2 +-
 integration-tests/worker/test/integration.test.ts | 15 +++++----------
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/integration-tests/worker/test/exit-reasons.test.ts b/integration-tests/worker/test/exit-reasons.test.ts
index 89712326c..f8e981276 100644
--- a/integration-tests/worker/test/exit-reasons.test.ts
+++ b/integration-tests/worker/test/exit-reasons.test.ts
@@ -77,7 +77,7 @@ test('exception: autoinstall error', async (t) => {
   );
 });
 
-test.only('kill: oom', async (t) => {
+test('kill: oom', async (t) => {
   const attempt = {
     id: crypto.randomUUID(),
     jobs: [
diff --git a/integration-tests/worker/test/integration.test.ts b/integration-tests/worker/test/integration.test.ts
index c4e202555..2fff278fc 100644
--- a/integration-tests/worker/test/integration.test.ts
+++ b/integration-tests/worker/test/integration.test.ts
@@ -327,8 +327,7 @@ test('blacklist a non-openfn adaptor', (t) => {
   });
 });
 
-// TODO this fails, I think this is a wider timeout problem but I don't really want to get into it now
-test.skip('a timeout error should still call run-complete', (t) => {
+test('a timeout error should still call run-complete', (t) => {
   return new Promise(async (done) => {
     const attempt = {
       id: crypto.randomUUID(),
@@ -345,12 +344,10 @@ test.skip('a timeout error should still call run-complete', (t) => {
     };
 
     lightning.once('run:complete', (event) => {
-      console.log(event.payload);
       t.is(event.payload.reason, 'kill');
     });
 
     lightning.once('attempt:complete', (event) => {
-      console.log(event.payload);
       done();
     });
 
@@ -358,7 +355,7 @@ test.skip('a timeout error should still call run-complete', (t) => {
   });
 });
 
-test.only('an OOM error should still call run-complete', (t) => {
+test('an OOM error should still call run-complete', (t) => {
   return new Promise(async (done) => {
     const attempt = {
       id: crypto.randomUUID(),
@@ -377,13 +374,11 @@ test.only('an OOM error should still call run-complete', (t) => {
       ],
     };
 
-    // lightning.once('run:complete', (event) => {
-    //   console.log(event.payload);
-    //   t.is(event.payload.reason, 'kill');
-    // });
+    lightning.once('run:complete', (event) => {
+      t.is(event.payload.reason, 'kill');
+    });
 
     lightning.once('attempt:complete', (event) => {
-      console.log(event.payload);
       done();
     });
 

From 544c145577ba5ea874eae7cce1063a1218a29829 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 30 Nov 2023 18:28:08 +0000
Subject: [PATCH 21/28] timeout errors to severity: kill

---
 packages/engine-multi/src/errors.ts   | 2 +-
 packages/runtime/src/errors.ts        | 2 +-
 packages/ws-worker/src/api/execute.ts | 4 +++-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/packages/engine-multi/src/errors.ts b/packages/engine-multi/src/errors.ts
index f26c0613c..2da37b270 100644
--- a/packages/engine-multi/src/errors.ts
+++ b/packages/engine-multi/src/errors.ts
@@ -9,7 +9,7 @@ export class EngineError extends Error {
 // This is thrown if a workflow takes too long to run
 // It is generated by workerpool and thrown if the workerpool promise fails to resolve
 export class TimeoutError extends EngineError {
-  severity = 'crash';
+  severity = 'kill';
   type = 'TimeoutError';
   duration;
   constructor(durationInMs: number) {
diff --git a/packages/runtime/src/errors.ts b/packages/runtime/src/errors.ts
index 60a70fbbd..01d157de8 100644
--- a/packages/runtime/src/errors.ts
+++ b/packages/runtime/src/errors.ts
@@ -225,7 +225,7 @@ export class SecurityError extends RTError {
 export class TimeoutError extends RTError {
   type = 'TimeoutError';
   name = 'TimeoutError';
-  severity = 'crash';
+  severity = 'kill';
   message: string;
   constructor(duration: number) {
     super();
diff --git a/packages/ws-worker/src/api/execute.ts b/packages/ws-worker/src/api/execute.ts
index ca94f6e0c..123942bd2 100644
--- a/packages/ws-worker/src/api/execute.ts
+++ b/packages/ws-worker/src/api/execute.ts
@@ -289,7 +289,7 @@ export async function onWorkflowError(
 
     // If there's a job still running, make sure it gets marked complete
     if (state.activeJob) {
-      await onJobError(context, event);
+      await onJobError(context, { error: event });
     }
 
     await sendEvent<AttemptCompletePayload>(channel, ATTEMPT_COMPLETE, {
@@ -301,6 +301,8 @@ export async function onWorkflowError(
   } catch (e: any) {
     logger.error('ERROR in workflow-error handler:', e.message);
     logger.error(e);
+
+    onFinish({});
   }
 }
 

From 6790f20b50b328de8790b66da31b322d4256bc93 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 10:58:47 +0000
Subject: [PATCH 22/28] remove .onlys

---
 packages/runtime/test/execute/plan.test.ts | 2 +-
 packages/ws-worker/test/lightning.test.ts  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/runtime/test/execute/plan.test.ts b/packages/runtime/test/execute/plan.test.ts
index 9f058e5f6..1cdd96682 100644
--- a/packages/runtime/test/execute/plan.test.ts
+++ b/packages/runtime/test/execute/plan.test.ts
@@ -801,7 +801,7 @@ test('keep executing after an error', async (t) => {
   t.falsy(result.x);
 });
 
-test.only('simple on-error handler', async (t) => {
+test('simple on-error handler', async (t) => {
   const plan: ExecutionPlan = {
     jobs: [
       {
diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts
index 8980e9d07..d2a8ec651 100644
--- a/packages/ws-worker/test/lightning.test.ts
+++ b/packages/ws-worker/test/lightning.test.ts
@@ -290,7 +290,7 @@ test.serial(
   }
 );
 
-test.serial.only(
+test.serial(
   `events: lightning should receive a ${e.RUN_COMPLETE} event even if the attempt fails`,
   (t) => {
     return new Promise((done) => {

From 346f607701c6964b30a757e9718b68bb8234f270 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 11:00:03 +0000
Subject: [PATCH 23/28] runtime: fix test

---
 packages/runtime/test/errors.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/runtime/test/errors.test.ts b/packages/runtime/test/errors.test.ts
index cdead8b28..90a9d9d16 100644
--- a/packages/runtime/test/errors.test.ts
+++ b/packages/runtime/test/errors.test.ts
@@ -16,7 +16,7 @@ test('crash on timeout', async (t) => {
   }
 
   t.truthy(error);
-  t.is(error.severity, 'crash');
+  t.is(error.severity, 'kill');
   t.is(error.type, 'TimeoutError');
   t.is(error.message, 'Job took longer than 1ms to complete');
 });

From 19f5a447ad59a1e6a3f18e949f74831cd36d86ba Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 11:49:33 +0000
Subject: [PATCH 24/28] lightning-mock: be more lenient with output dataclips

---
 packages/lightning-mock/src/api-sockets.ts | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/packages/lightning-mock/src/api-sockets.ts b/packages/lightning-mock/src/api-sockets.ts
index 96f1752d4..763c3f06c 100644
--- a/packages/lightning-mock/src/api-sockets.ts
+++ b/packages/lightning-mock/src/api-sockets.ts
@@ -419,21 +419,11 @@ const createSocketAPI = (
 
     let payload: any = validateReasons(evt.payload);
 
-    if (!output_dataclip) {
-      payload = {
-        status: 'error',
-        response: 'no output_dataclip',
-      };
-    } else if (output_dataclip_id) {
+    if (output_dataclip_id && output_dataclip) {
       if (!state.dataclips) {
         state.dataclips = {};
       }
       state.dataclips[output_dataclip_id] = JSON.parse(output_dataclip!);
-    } else {
-      payload = {
-        status: 'error',
-        response: 'no output_dataclip_id',
-      };
     }
 
     // be polite and acknowledge the event

From 6a72388c26ecab379ec8a0b3ad71fdd0407f1e92 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 12:08:18 +0000
Subject: [PATCH 25/28] worker: fix tests

---
 packages/ws-worker/test/api/execute.test.ts | 10 ++++------
 packages/ws-worker/test/lightning.test.ts   |  1 -
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/packages/ws-worker/test/api/execute.test.ts b/packages/ws-worker/test/api/execute.test.ts
index 3e57990b1..de482e50f 100644
--- a/packages/ws-worker/test/api/execute.test.ts
+++ b/packages/ws-worker/test/api/execute.test.ts
@@ -403,18 +403,16 @@ test('workflowError should trigger runComplete with a reason', async (t) => {
   const channel = mockChannel({
     [RUN_COMPLETE]: (evt) => {
       t.is(evt.reason, 'crash');
+      t.is(evt.error_message, 'it crashed');
       return true;
     },
     [ATTEMPT_COMPLETE]: () => true,
   });
 
   const event = {
-    error: {
-      severity: 'crash',
-      type: 'Err',
-      message: 'it crashed',
-    },
-    state: {},
+    severity: 'crash',
+    type: 'Err',
+    message: 'it crashed',
   };
 
   const context = { channel, state, onFinish: () => {} };
diff --git a/packages/ws-worker/test/lightning.test.ts b/packages/ws-worker/test/lightning.test.ts
index d2a8ec651..b110bcc85 100644
--- a/packages/ws-worker/test/lightning.test.ts
+++ b/packages/ws-worker/test/lightning.test.ts
@@ -304,7 +304,6 @@ test.serial(
       ]);
 
       lng.onSocketEvent(e.RUN_COMPLETE, attempt.id, ({ payload }) => {
-        console.log(payload);
         t.not(payload.reason, 'success');
         t.pass('called run complete');
       });

From eb24a585841db3c93f754f4ee10ab46f8590c142 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 1 Dec 2023 12:18:16 +0000
Subject: [PATCH 26/28] lightning-mock: remove unneeded tests

---
 .../test/events/run-complete.test.ts          | 40 -------------------
 1 file changed, 40 deletions(-)

diff --git a/packages/lightning-mock/test/events/run-complete.test.ts b/packages/lightning-mock/test/events/run-complete.test.ts
index 3b4110696..e8aad4be0 100644
--- a/packages/lightning-mock/test/events/run-complete.test.ts
+++ b/packages/lightning-mock/test/events/run-complete.test.ts
@@ -52,46 +52,6 @@ test.serial('save dataclip id to state', async (t) => {
   });
 });
 
-test.serial('error if no dataclip', async (t) => {
-  return new Promise(async (done) => {
-    const attempt = createAttempt();
-
-    server.startAttempt(attempt.id);
-
-    const event = {
-      reason: 'success',
-      output_dataclip: null,
-      output_dataclip_id: 't',
-    };
-    const channel = await join(client, attempt.id);
-
-    channel.push(RUN_COMPLETE, event).receive('error', () => {
-      t.pass('event rejected');
-      done();
-    });
-  });
-});
-
-test.serial('error if no dataclip_d', async (t) => {
-  return new Promise(async (done) => {
-    const attempt = createAttempt();
-
-    server.startAttempt(attempt.id);
-
-    const event = {
-      reason: 'success',
-      output_dataclip: JSON.stringify({ x: 22 }),
-      output_dataclip_id: undefined,
-    };
-    const channel = await join(client, attempt.id);
-
-    channel.push(RUN_COMPLETE, event).receive('error', () => {
-      t.pass('event rejected');
-      done();
-    });
-  });
-});
-
 test.serial('error if no reason', async (t) => {
   return new Promise(async (done) => {
     const attempt = createAttempt();

From 25fe0baad86d503a3247c5712d57356d5665661e Mon Sep 17 00:00:00 2001
From: Taylor Downs <downs.taylor@gmail.com>
Date: Sat, 2 Dec 2023 21:23:44 -0500
Subject: [PATCH 27/28] run pnpm i

---
 pnpm-lock.yaml | 110 +++++--------------------------------------------
 1 file changed, 10 insertions(+), 100 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 9bf56ec71..7fbea3863 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -146,18 +146,6 @@ importers:
         specifier: ^5.1.6
         version: 5.1.6
 
-  integration-tests/worker/tmp/repo/bench:
-    dependencies:
-      '@openfn/language-common_1.7.0':
-        specifier: npm:@openfn/language-common@^1.7.0
-        version: /@openfn/language-common@1.7.5
-
-  integration-tests/worker/tmp/repo/exit-reason:
-    dependencies:
-      '@openfn/language-common_latest':
-        specifier: npm:@openfn/language-common@^1.11.1
-        version: /@openfn/language-common@1.11.1
-
   packages/cli:
     dependencies:
       '@inquirer/prompts':
@@ -424,10 +412,6 @@ importers:
         specifier: ^5.1.6
         version: 5.1.6
 
-  packages/engine-multi/tmp/a/b/c: {}
-
-  packages/engine-multi/tmp/repo: {}
-
   packages/lightning-mock:
     dependencies:
       '@koa/router':
@@ -1342,11 +1326,6 @@ packages:
       heap: 0.2.7
     dev: false
 
-  /@fastify/busboy@2.1.0:
-    resolution: {integrity: sha512-+KpH+QxZU7O4675t3mnkQKcZZg56u+K/Ct2K+N2AZYNVK8kyeo/bI18tI8aPm3tvNNRyTWfj6s5tnGNlcbQRsA==}
-    engines: {node: '>=14'}
-    dev: false
-
   /@inquirer/checkbox@1.3.5:
     resolution: {integrity: sha512-ZznkPU+8XgNICKkqaoYENa0vTw9jeToEHYyG5gUKpGmY+4PqPTsvLpSisOt9sukLkYzPRkpSCHREgJLqbCG3Fw==}
     engines: {node: '>=14.18.0'}
@@ -1603,21 +1582,6 @@ packages:
       semver: 7.5.4
     dev: true
 
-  /@openfn/language-common@1.11.1:
-    resolution: {integrity: sha512-pyi2QymdF9NmUYJX/Bsv5oBy7TvzICfKcnCqutq412HYq2KTGKDO2dMWloDrxrH1kuzG+4XkSn0ZUom36b3KAA==}
-    dependencies:
-      ajv: 8.12.0
-      axios: 1.1.3
-      csv-parse: 5.5.2
-      csvtojson: 2.0.10
-      date-fns: 2.30.0
-      jsonpath-plus: 4.0.0
-      lodash: 4.17.21
-      undici: 5.27.2
-    transitivePeerDependencies:
-      - debug
-    dev: false
-
   /@openfn/language-common@1.7.5:
     resolution: {integrity: sha512-QivV3v5Oq5fb4QMopzyqUUh+UGHaFXBdsGr6RCmu6bFnGXdJdcQ7GpGpW5hKNq29CkmE23L/qAna1OLr4rP/0w==}
     dependencies:
@@ -1627,6 +1591,7 @@ packages:
       lodash: 4.17.21
     transitivePeerDependencies:
       - debug
+    dev: true
 
   /@openfn/language-common@2.0.0-rc3:
     resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==}
@@ -2078,15 +2043,6 @@ packages:
       clean-stack: 4.2.0
       indent-string: 5.0.0
 
-  /ajv@8.12.0:
-    resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==}
-    dependencies:
-      fast-deep-equal: 3.1.3
-      json-schema-traverse: 1.0.0
-      require-from-string: 2.0.2
-      uri-js: 4.4.1
-    dev: false
-
   /ansi-colors@4.1.3:
     resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==}
     engines: {node: '>=6'}
@@ -2247,6 +2203,7 @@ packages:
 
   /asynckit@0.4.0:
     resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+    dev: true
 
   /atob@2.1.2:
     resolution: {integrity: sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==}
@@ -2394,6 +2351,7 @@ packages:
       proxy-from-env: 1.1.0
     transitivePeerDependencies:
       - debug
+    dev: true
 
   /b4a@1.6.1:
     resolution: {integrity: sha512-AsKjNhz72yxteo/0EtQEiwkMUgk/tGmycXlbG4g3Ard2/ULtNLUykGOkeK0egmN27h0xMAhb76jYccW+XTBExA==}
@@ -2474,10 +2432,6 @@ packages:
       readable-stream: 4.2.0
     dev: true
 
-  /bluebird@3.7.2:
-    resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
-    dev: false
-
   /blueimp-md5@2.19.0:
     resolution: {integrity: sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==}
 
@@ -2874,6 +2828,7 @@ packages:
     engines: {node: '>= 0.8'}
     dependencies:
       delayed-stream: 1.0.0
+    dev: true
 
   /commander@4.1.1:
     resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==}
@@ -3016,10 +2971,6 @@ packages:
     resolution: {integrity: sha512-cO1I/zmz4w2dcKHVvpCr7JVRu8/FymG5OEpmvsZYlccYolPBLoVGKUHgNoc4ZGkFeFlWGEDmMyBM+TTqRdW/wg==}
     dev: true
 
-  /csv-parse@5.5.2:
-    resolution: {integrity: sha512-YRVtvdtUNXZCMyK5zd5Wty1W6dNTpGKdqQd4EQ8tl/c6KW1aMBB1Kg1ppky5FONKmEqGJ/8WjLlTNLPne4ioVA==}
-    dev: false
-
   /csv-stringify@5.6.5:
     resolution: {integrity: sha512-PjiQ659aQ+fUTQqSrd1XEDnOr52jh30RBurfzkscaE2tPaFsDH5wOAHJiw8XAHphRknCwMUE9KRayc4K/NbO8A==}
     dev: true
@@ -3034,16 +2985,6 @@ packages:
       stream-transform: 2.1.3
     dev: true
 
-  /csvtojson@2.0.10:
-    resolution: {integrity: sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==}
-    engines: {node: '>=4.0.0'}
-    hasBin: true
-    dependencies:
-      bluebird: 3.7.2
-      lodash: 4.17.21
-      strip-bom: 2.0.0
-    dev: false
-
   /currently-unhandled@0.4.1:
     resolution: {integrity: sha512-/fITjgjGU50vjQ4FH6eUoYu+iUoUKIXws2hL15JJpIR+BbTxaXQsMuuyjtNh2WqsSBS5nsaZHFsFecyw5CCAng==}
     engines: {node: '>=0.10.0'}
@@ -3184,6 +3125,7 @@ packages:
   /delayed-stream@1.0.0:
     resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
     engines: {node: '>=0.4.0'}
+    dev: true
 
   /delegates@1.0.0:
     resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
@@ -4009,10 +3951,6 @@ packages:
       - supports-color
     dev: true
 
-  /fast-deep-equal@3.1.3:
-    resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==}
-    dev: false
-
   /fast-diff@1.3.0:
     resolution: {integrity: sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw==}
 
@@ -4159,6 +4097,7 @@ packages:
     peerDependenciesMeta:
       debug:
         optional: true
+    dev: true
 
   /for-in@1.0.2:
     resolution: {integrity: sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==}
@@ -4189,6 +4128,7 @@ packages:
       asynckit: 0.4.0
       combined-stream: 1.0.8
       mime-types: 2.1.35
+    dev: true
 
   /fragment-cache@0.2.1:
     resolution: {integrity: sha512-GMBAbW9antB8iZRHLoGw0b3HANt57diZYFO/HL1JGIC1MjKrdmhxvrJbupnVvpys0zsz7yBApXdQyfepKly2kA==}
@@ -4971,10 +4911,6 @@ packages:
     resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==}
     engines: {node: '>=12'}
 
-  /is-utf8@0.2.1:
-    resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
-    dev: false
-
   /is-weakref@1.0.2:
     resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==}
     dependencies:
@@ -5068,10 +5004,6 @@ packages:
     resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
     dev: true
 
-  /json-schema-traverse@1.0.0:
-    resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==}
-    dev: false
-
   /jsonfile@4.0.0:
     resolution: {integrity: sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==}
     optionalDependencies:
@@ -5086,6 +5018,7 @@ packages:
   /jsonpath-plus@4.0.0:
     resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==}
     engines: {node: '>=10.0'}
+    dev: true
 
   /jsonpath@1.1.1:
     resolution: {integrity: sha512-l6Cg7jRpixfbgoWgkrl77dgEj8RPvND0wMH6TwQmi9Qs4TFfS9u5cUFnbeKTwj5ga5Y3BTGGNI28k117LJ009w==}
@@ -6392,6 +6325,7 @@ packages:
 
   /proxy-from-env@1.1.0:
     resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
+    dev: true
 
   /proxy-middleware@0.15.0:
     resolution: {integrity: sha512-EGCG8SeoIRVMhsqHQUdDigB2i7qU7fCsWASwn54+nPutYO8n4q6EiwMzyfWlC+dzRFExP+kvcnDFdBDHoZBU7Q==}
@@ -6424,6 +6358,7 @@ packages:
   /punycode@2.3.0:
     resolution: {integrity: sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==}
     engines: {node: '>=6'}
+    dev: true
 
   /qs@6.11.2:
     resolution: {integrity: sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==}
@@ -6629,11 +6564,6 @@ packages:
     resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==}
     engines: {node: '>=0.10.0'}
 
-  /require-from-string@2.0.2:
-    resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==}
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   /require-main-filename@2.0.0:
     resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==}
     dev: true
@@ -7157,13 +7087,6 @@ packages:
     dependencies:
       ansi-regex: 6.0.1
 
-  /strip-bom@2.0.0:
-    resolution: {integrity: sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g==}
-    engines: {node: '>=0.10.0'}
-    dependencies:
-      is-utf8: 0.2.1
-    dev: false
-
   /strip-bom@3.0.0:
     resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
     engines: {node: '>=4'}
@@ -7717,13 +7640,6 @@ packages:
     resolution: {integrity: sha512-hEQt0+ZLDVUMhebKxL4x1BTtDY7bavVofhZ9KZ4aI26X9SRaE+Y3m83XUL1UP2jn8ynjndwCCpEHdUG+9pP1Tw==}
     dev: true
 
-  /undici@5.27.2:
-    resolution: {integrity: sha512-iS857PdOEy/y3wlM3yRp+6SNQQ6xU0mmZcwRSriqk+et/cwWAtwmIGf6WkoDN2EK/AMdCO/dfXzIwi+rFMrjjQ==}
-    engines: {node: '>=14.0'}
-    dependencies:
-      '@fastify/busboy': 2.1.0
-    dev: false
-
   /union-value@1.0.1:
     resolution: {integrity: sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==}
     engines: {node: '>=0.10.0'}
@@ -7774,12 +7690,6 @@ packages:
     engines: {node: '>=4'}
     dev: true
 
-  /uri-js@4.4.1:
-    resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==}
-    dependencies:
-      punycode: 2.3.0
-    dev: false
-
   /urix@0.1.0:
     resolution: {integrity: sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg==}
     deprecated: Please see https://github.com/lydell/urix#deprecated

From 2f10a27c0eec009db7743bbd03d4e1b11849cc7e Mon Sep 17 00:00:00 2001
From: Taylor Downs <downs.taylor@gmail.com>
Date: Sat, 2 Dec 2023 21:27:47 -0500
Subject: [PATCH 28/28] bump version (and fix readme for release)

---
 .changeset/moody-mice-swim.md         |  5 -----
 .changeset/twenty-oranges-push.md     |  5 -----
 .changeset/violet-ligers-walk.md      |  5 -----
 README.md                             |  2 +-
 integration-tests/worker/CHANGELOG.md | 11 +++++++++++
 integration-tests/worker/package.json |  2 +-
 packages/engine-multi/CHANGELOG.md    |  7 +++++++
 packages/engine-multi/package.json    |  2 +-
 packages/lightning-mock/CHANGELOG.md  |  8 ++++++++
 packages/lightning-mock/package.json  |  2 +-
 packages/ws-worker/CHANGELOG.md       |  9 +++++++++
 packages/ws-worker/package.json       |  2 +-
 12 files changed, 40 insertions(+), 20 deletions(-)
 delete mode 100644 .changeset/moody-mice-swim.md
 delete mode 100644 .changeset/twenty-oranges-push.md
 delete mode 100644 .changeset/violet-ligers-walk.md

diff --git a/.changeset/moody-mice-swim.md b/.changeset/moody-mice-swim.md
deleted file mode 100644
index cb8d605ee..000000000
--- a/.changeset/moody-mice-swim.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@openfn/ws-worker': patch
----
-
-Ensure capacity is also set on the engine
diff --git a/.changeset/twenty-oranges-push.md b/.changeset/twenty-oranges-push.md
deleted file mode 100644
index b5a97b3fb..000000000
--- a/.changeset/twenty-oranges-push.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@openfn/engine-multi': patch
----
-
-Handle async errors in the runtime
diff --git a/.changeset/violet-ligers-walk.md b/.changeset/violet-ligers-walk.md
deleted file mode 100644
index 623d0e174..000000000
--- a/.changeset/violet-ligers-walk.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@openfn/engine-multi': patch
----
-
-Throw a better error on process.exit
diff --git a/README.md b/README.md
index c500d611b..40962dbb0 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ New releases will be published to npm automatically when merging into main.
 
 Before merging to main, check out the release branch locally and run the following steps:
 
-1. Run `pnpm changeset` from root to bump versions
+1. Run `pnpm changeset version` from root to bump versions
 1. Run `pnpm install`
 1. Commit the new version numbers
 1. Run `pnpm changeset tag` to generate tags
diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md
index e49464cd4..c76b78f9c 100644
--- a/integration-tests/worker/CHANGELOG.md
+++ b/integration-tests/worker/CHANGELOG.md
@@ -1,5 +1,16 @@
 # @openfn/integration-tests-worker
 
+## 1.0.23
+
+### Patch Changes
+
+- Updated dependencies [6c3e9e42]
+- Updated dependencies [05ccc10b]
+- Updated dependencies [7235bf5e]
+  - @openfn/ws-worker@0.2.12
+  - @openfn/engine-multi@0.2.3
+  - @openfn/lightning-mock@1.1.5
+
 ## 1.0.22
 
 ### Patch Changes
diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json
index 21c16cd39..f76a2e708 100644
--- a/integration-tests/worker/package.json
+++ b/integration-tests/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.22",
+  "version": "1.0.23",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md
index a5f7ac5e7..cf44a0559 100644
--- a/packages/engine-multi/CHANGELOG.md
+++ b/packages/engine-multi/CHANGELOG.md
@@ -1,5 +1,12 @@
 # engine-multi
 
+## 0.2.3
+
+### Patch Changes
+
+- 05ccc10b: Handle async errors in the runtime
+- 7235bf5e: Throw a better error on process.exit
+
 ## 0.2.2
 
 ### Patch Changes
diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json
index c1774f152..cfb05a6d8 100644
--- a/packages/engine-multi/package.json
+++ b/packages/engine-multi/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "0.2.2",
+  "version": "0.2.3",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",
diff --git a/packages/lightning-mock/CHANGELOG.md b/packages/lightning-mock/CHANGELOG.md
index 51d8931c1..68685b49b 100644
--- a/packages/lightning-mock/CHANGELOG.md
+++ b/packages/lightning-mock/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @openfn/lightning-mock
 
+## 1.1.5
+
+### Patch Changes
+
+- Updated dependencies [05ccc10b]
+- Updated dependencies [7235bf5e]
+  - @openfn/engine-multi@0.2.3
+
 ## 1.1.4
 
 ### Patch Changes
diff --git a/packages/lightning-mock/package.json b/packages/lightning-mock/package.json
index da77fc87e..364d52610 100644
--- a/packages/lightning-mock/package.json
+++ b/packages/lightning-mock/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/lightning-mock",
-  "version": "1.1.4",
+  "version": "1.1.5",
   "private": true,
   "description": "A mock Lightning server",
   "main": "dist/index.js",
diff --git a/packages/ws-worker/CHANGELOG.md b/packages/ws-worker/CHANGELOG.md
index ee5dacbea..3c5ca392a 100644
--- a/packages/ws-worker/CHANGELOG.md
+++ b/packages/ws-worker/CHANGELOG.md
@@ -1,5 +1,14 @@
 # ws-worker
 
+## 0.2.12
+
+### Patch Changes
+
+- 6c3e9e42: Ensure capacity is also set on the engine
+- Updated dependencies [05ccc10b]
+- Updated dependencies [7235bf5e]
+  - @openfn/engine-multi@0.2.3
+
 ## 0.2.11
 
 ### Patch Changes
diff --git a/packages/ws-worker/package.json b/packages/ws-worker/package.json
index a14630f98..c1ad64c7f 100644
--- a/packages/ws-worker/package.json
+++ b/packages/ws-worker/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/ws-worker",
-  "version": "0.2.11",
+  "version": "0.2.12",
   "description": "A Websocket Worker to connect Lightning to a Runtime Engine",
   "main": "dist/index.js",
   "type": "module",