Merge pull request #559 from OpenFn/release/next

Release [email protected] [email protected]
OpenFn · Jan 11, 2024 · a4b83f6 · a4b83f6
2 parents 8005133 + c92e550
commit a4b83f6
Show file tree

Hide file tree

Showing 33 changed files with 279 additions and 45 deletions.
diff --git a/integration-tests/worker/CHANGELOG.md b/integration-tests/worker/CHANGELOG.md
@@ -1,5 +1,21 @@
 # @openfn/integration-tests-worker
 
+## 1.0.29
+
+### Patch Changes
+
+- Updated dependencies
+  - @openfn/ws-worker@0.5.0
+
+## 1.0.28
+
+### Patch Changes
+
+- Updated dependencies [5c45e1e]
+  - @openfn/engine-multi@0.2.6
+  - @openfn/lightning-mock@1.1.8
+  - @openfn/ws-worker@0.4.1
+
 ## 1.0.27
 
 ### Patch Changes

diff --git a/integration-tests/worker/package.json b/integration-tests/worker/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.27",
+  "version": "1.0.29",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

diff --git a/integration-tests/worker/test/attempts.test.ts b/integration-tests/worker/test/attempts.test.ts
@@ -187,14 +187,15 @@ test.serial('run parallel jobs', async (t) => {
 test('run a http adaptor job', async (t) => {
   const job = createJob({
     adaptor: '@openfn/[email protected]',
-    body: 'get("https://jsonplaceholder.typicode.com/todos/1");',
+    body: `get("https://jsonplaceholder.typicode.com/todos/1");
+    fn((state) => { state.res = state.response; return state });`,
   });
   const attempt = createAttempt([], [job], []);
   const result = await run(t, attempt);
 
-  t.truthy(result.response);
-  t.is(result.response.status, 200);
-  t.truthy(result.response.headers);
+  t.truthy(result.res);
+  t.is(result.res.status, 200);
+  t.truthy(result.res.headers);
 
   t.deepEqual(result.data, {
     userId: 1,

diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
@@ -1,5 +1,14 @@
 # @openfn/cli
 
+## 0.4.13
+
+### Patch Changes
+
+- Updated dependencies [3f0010e]
+- Updated dependencies [56b6e44]
+  - @openfn/deploy@0.3.0
+  - @openfn/runtime@0.2.4
+
 ## 0.4.12
 
 ### Patch Changes

diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "0.4.12",
+  "version": "0.4.13",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @openfn/deploy
 
+## 0.3.0
+
+### Minor Changes
+
+- 3f0010e: add support for javascript edge conditions
+
 ## 0.2.10
 
 ### Patch Changes

diff --git a/packages/deploy/package.json b/packages/deploy/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/deploy",
-  "version": "0.2.10",
+  "version": "0.3.0",
   "description": "Deploy projects to Lightning instances",
   "type": "module",
   "exports": {

diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts
@@ -143,11 +143,13 @@ function mergeEdges(
           const edge: StateEdge = assignIfTruthy(
             {
               id,
-              condition: specEdge.condition ?? null,
+              condition_type: specEdge.condition_type ?? null,
               target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '',
               enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true),
             },
             {
+              condition_expression: specEdge.condition_expression,
+              condition_label: specEdge.condition_label,
               source_job_id: jobs[specEdge.source_job ?? -1]?.id,
               source_trigger_id: triggers[specEdge.source_trigger ?? -1]?.id,
             }

diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts
@@ -16,15 +16,19 @@ export type Trigger = {
 
 export type StateEdge = {
   id: string;
-  condition: string | null;
+  condition_type: string;
+  condition_expression: string | null;
+  condition_label: string | null;
   source_job_id: string | null;
   source_trigger_id: string | null;
   target_job_id: string;
   enabled?: boolean;
 };
 
 export type SpecEdge = {
-  condition: string | null;
+  condition_type: string;
+  condition_expression: string | null;
+  condition_label: string | null;
   source_job?: string | null;
   source_trigger?: string | null;
   target_job: string | null;

diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts
@@ -28,10 +28,14 @@ export function fullExampleSpec() {
         },
         edges: {
           'trigger-one->job-a': {
+            condition_type: 'always',
             source_trigger: 'trigger-one',
             target_job: 'job-a',
           },
           'job-a->job-b': {
+            condition_type: 'js_expression',
+            condition_expression: 'state.data > 18',
+            condition_label: 'not-minor',
             source_job: 'job-a',
             target_job: 'job-b',
           },
@@ -75,14 +79,16 @@ export function fullExampleState() {
         edges: {
           'trigger-one->job-a': {
             id: 'ea264d6d-9767-4a2c-810f-deb10961a6dc',
-            condition: null,
+            condition_type: 'always',
             source_trigger_id: '71f0cbf1-4d8e-443e-afca-8a479ec281a1',
             target_job_id: '68e172b8-1cca-4085-aadf-8534761ef7c2',
             enabled: true,
           },
           'job-a->job-b': {
             id: '7132f768-e8e8-4167-8fc2-8d422244281f',
-            condition: null,
+            condition_type: 'js_expression',
+            condition_expression: 'state.data > 18',
+            condition_label: 'not-minor',
             source_job_id: '68e172b8-1cca-4085-aadf-8534761ef7c2',
             target_job_id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37',
             enabled: true,
@@ -114,27 +120,27 @@ export const lightningProjectPayload = {
           id: '7d8262a9-3bfa-4cdc-b562-c0a04c83c572',
           source_trigger_id: '951fb278-3829-40e6-b86d-c5a6603a0df1',
           target_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc',
-          condition: 'always',
+          condition_type: 'always',
           enabled: true,
         },
         {
           id: 'a571d495-8f47-4c24-9be4-631eff6e3b8d',
           target_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
-          condition: 'on_job_success',
+          condition_type: 'on_job_success',
           source_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc',
           enabled: true,
         },
         {
           id: 'e4a2d2ff-1281-4549-b919-5a6fd369bdc3',
           target_job_id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b',
-          condition: 'on_job_success',
+          condition_type: 'on_job_success',
           source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           enabled: false,
         },
         {
           id: 'f7163a97-03c5-4a45-9abf-69f1b771655f',
           target_job_id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac',
-          condition: 'on_job_failure',
+          condition_type: 'on_job_failure',
           source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           enabled: true,
         },
@@ -184,7 +190,7 @@ export const lightningProjectPayload = {
           id: '6e0d23fe-d3eb-418f-b5f8-32da9488baec',
           source_trigger_id: '388bbb05-9a88-4493-9ef1-7404274c27b8',
           target_job_id: '74306d89-2324-4292-9cd4-99450b750050',
-          condition: 'always',
+          condition_type: 'always',
           enabled: true,
         },
       ],
@@ -227,27 +233,27 @@ export const lightningProjectState = {
           id: '7d8262a9-3bfa-4cdc-b562-c0a04c83c572',
           source_trigger_id: '951fb278-3829-40e6-b86d-c5a6603a0df1',
           target_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc',
-          condition: 'always',
+          condition_type: 'always',
           enabled: true,
         },
         'FHIR-standard-Data-with-change->Send-to-OpenHIM-to-route-to-SHR': {
           id: 'a571d495-8f47-4c24-9be4-631eff6e3b8d',
           target_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
-          condition: 'on_job_success',
+          condition_type: 'on_job_success',
           source_job_id: '8852a349-0936-4141-8c12-d1bfd910e2dc',
           enabled: true,
         },
         'Send-to-OpenHIM-to-route-to-SHR->Notify-CHW-upload-successful': {
           id: 'e4a2d2ff-1281-4549-b919-5a6fd369bdc3',
           target_job_id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b',
-          condition: 'on_job_success',
+          condition_type: 'on_job_success',
           source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           enabled: false,
         },
         'Send-to-OpenHIM-to-route-to-SHR->Notify-CHW-upload-failed': {
           id: 'f7163a97-03c5-4a45-9abf-69f1b771655f',
           target_job_id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac',
-          condition: 'on_job_failure',
+          condition_type: 'on_job_failure',
           source_job_id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           enabled: true,
         },
@@ -297,7 +303,7 @@ export const lightningProjectState = {
           id: '6e0d23fe-d3eb-418f-b5f8-32da9488baec',
           source_trigger_id: '388bbb05-9a88-4493-9ef1-7404274c27b8',
           target_job_id: '74306d89-2324-4292-9cd4-99450b750050',
-          condition: 'always',
+          condition_type: 'always',
           enabled: true,
         },
       },

diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts
@@ -143,14 +143,16 @@ test('toNextState with empty state', (t) => {
         edges: {
           'trigger-one->job-a': {
             id: getItem(result, 'edges', 'trigger-one->job-a').id,
-            condition: null,
+            condition_type: 'always',
             source_trigger_id: getItem(result, 'triggers', 'trigger-one').id,
             target_job_id: getItem(result, 'jobs', 'job-a').id,
             enabled: true,
           },
           'job-a->job-b': {
             id: getItem(result, 'edges', 'job-a->job-b').id,
-            condition: null,
+            condition_type: 'js_expression',
+            condition_expression: 'state.data > 18',
+            condition_label: 'not-minor',
             source_job_id: getItem(result, 'jobs', 'job-a').id,
             target_job_id: getItem(result, 'jobs', 'job-b').id,
             enabled: true,
@@ -433,7 +435,7 @@ test('getStateFromProjectPayload with minimal project', (t) => {
           {
             id: 't1-job-1',
             target_job_id: 'job-1',
-            condition: 'on_job_failure',
+            condition_type: 'on_job_failure',
             source_trigger_id: 't1',
             enabled: true,
           },
@@ -469,7 +471,7 @@ test('getStateFromProjectPayload with minimal project', (t) => {
           'webhook->My-job': {
             id: 't1-job-1',
             target_job_id: 'job-1',
-            condition: 'on_job_failure',
+            condition_type: 'on_job_failure',
             source_trigger_id: 't1',
             enabled: true,
           },

diff --git a/packages/engine-multi/CHANGELOG.md b/packages/engine-multi/CHANGELOG.md
@@ -1,5 +1,13 @@
 # engine-multi
 
+## 0.2.6
+
+### Patch Changes
+
+- 5c45e1e: Remove response key from state
+- Updated dependencies [56b6e44]
+  - @openfn/runtime@0.2.4
+
 ## 0.2.5
 
 ### Patch Changes

diff --git a/packages/engine-multi/package.json b/packages/engine-multi/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/engine-multi",
-  "version": "0.2.5",
+  "version": "0.2.6",
   "description": "Multi-process runtime engine",
   "main": "dist/index.js",
   "type": "module",

diff --git a/packages/engine-multi/src/api.ts b/packages/engine-multi/src/api.ts
@@ -70,12 +70,19 @@ const createAPI = async function (
     memoryLimitMb: options.memoryLimitMb || DEFAULT_MEMORY_LIMIT,
 
     purge: options.hasOwnProperty('purge') ? options.purge : true,
+
+    statePropsToRemove: options.statePropsToRemove ?? [
+      'configuration',
+      'response',
+    ],
   };
   logger.info(`memory limit set to ${options.memoryLimitMb}mb`);
+  logger.info(`statePropsToRemove set to: `, engineOptions.statePropsToRemove);
 
   const engine = await createEngine(engineOptions);
 
   return {
+    options: engineOptions,
     version: pkg.version,
     execute: engine.execute,
     listen: engine.listen,

diff --git a/packages/engine-multi/src/api/execute.ts b/packages/engine-multi/src/api/execute.ts
@@ -49,6 +49,7 @@ const execute = async (context: ExecutionContext) => {
     const runOptions = {
       adaptorPaths,
       whitelist: options.whitelist,
+      statePropsToRemove: options.statePropsToRemove,
     };
 
     const events = {

diff --git a/packages/engine-multi/src/engine.ts b/packages/engine-multi/src/engine.ts
@@ -82,6 +82,8 @@ export type EngineOptions = {
   timeout?: number;
 
   purge?: boolean;
+
+  statePropsToRemove?: string[];
 };
 
 export type ExecuteOptions = {

diff --git a/packages/engine-multi/src/types.ts b/packages/engine-multi/src/types.ts
@@ -71,6 +71,8 @@ export interface EngineAPI extends EventEmitter {
 export interface RuntimeEngine {
   version: string;
 
+  options: EngineOptions;
+
   // TODO should return an unsubscribe hook
   listen(attemptId: string, listeners: any): void;
 

diff --git a/packages/engine-multi/src/worker/worker.ts b/packages/engine-multi/src/worker/worker.ts
@@ -21,6 +21,7 @@ type RunOptions = {
   adaptorPaths: Record<string, { path: string }>;
   whitelist?: RegExp[];
   sanitize: SanitizePolicies;
+  statePropsToRemove?: string[];
   // TODO timeout
 };
 
@@ -29,7 +30,8 @@ workerpool.worker({
   handshake: () => true,
 
   run: (plan: ExecutionPlan, runOptions: RunOptions) => {
-    const { adaptorPaths, whitelist, sanitize } = runOptions;
+    const { adaptorPaths, whitelist, sanitize, statePropsToRemove } =
+      runOptions;
     const { logger, jobLogger } = createLoggers(plan.id!, sanitize);
     const options = {
       strict: false,
@@ -40,6 +42,7 @@ workerpool.worker({
         whitelist,
         cacheKey: plan.id,
       },
+      statePropsToRemove,
       callbacks: {
         // TODO: this won't actually work across the worker boundary
         // For now I am preloading credentials