Merge pull request #659 from OpenFn/release/next

Release

examples/dts-inspector/CHANGELOG.md

@@ -1,5 +1,12 @@
 # dts-inspector
 
+## 1.0.16
+
+### Patch Changes
+
+- Updated dependencies [4deb5d4]
+  - @openfn/describe-package@0.0.19
+
 ## 1.0.15
 
 ### Patch Changes

examples/dts-inspector/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dts-inspector",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "",
   "main": "index.js",
   "type": "module",

integration-tests/cli/test/errors.test.ts

@@ -121,7 +121,7 @@ test.serial('multiple inputs', async (t) => {
   t.regex(error.message[0].message, /multiple dependencies detected for: c/i);
 });
 
-test.serial('invalid start', async (t) => {
+test.serial('invalid start on workflow (not found)', async (t) => {
   const { stdout, err } = await run(
     `openfn ${jobsPath}/invalid-start.json --log-json`
   );
@@ -130,9 +130,21 @@ test.serial('invalid start', async (t) => {
   const stdlogs = extractLogs(stdout);
 
   assertLog(t, stdlogs, /Error validating execution plan/i);
-  assertLog(t, stdlogs, /Workflow failed/i);
+  assertLog(t, stdlogs, /aborting/i);
 
-  // Find the error obejct which is logged out
   const error = stdlogs.find((l) => l.message[0].name === 'ValidationError');
-  t.regex(error.message[0].message, /could not find start job: nope/i);
+  t.regex(error.message[0].message, /Could not find start job: nope/i);
+});
+
+test.serial('invalid end (ambiguous)', async (t) => {
+  // Note that the start should override
+  const { stdout, err } = await run(
+    `openfn ${jobsPath}/invalid-start.json --log-json --start x1 --end x`
+  );
+  t.is(err.code, 1);
+
+  const stdlogs = extractLogs(stdout);
+
+  assertLog(t, stdlogs, /Error: end pattern matched multiple steps/i);
+  assertLog(t, stdlogs, /aborting/i);
 });

integration-tests/cli/test/execute-workflow.test.ts

@@ -148,7 +148,7 @@ test.serial(
   }
 );
 
-test.serial.only(
+test.serial(
   `openfn ${jobsPath}/wf-errors.json -iS "{ \\"data\\": { \\"number\\": 32 } }"`,
   async (t) => {
     const { err } = await run(t.title);
@@ -176,3 +176,15 @@ test.serial.only(
     });
   }
 );
+
+// export issues https://github.com/OpenFn/kit/issues/238
+test.serial(
+  `openfn ${jobsPath}/common-date.json -s ${jobsPath}/common-date-input.json`,
+  async (t) => {
+    const { err } = await run(t.title);
+    t.falsy(err);
+
+    const out = getJSON();
+    t.deepEqual(out, { data: '01/01/2024', result: '1/1/2024, 12:00:00 AM' })
+  }
+);

integration-tests/cli/test/fixtures/common-date-input.json

@@ -0,0 +1,3 @@
+{
+  "data": "01/01/2024"
+}

integration-tests/cli/test/fixtures/common-date.json

@@ -0,0 +1,10 @@
+{
+  "workflow": {
+    "steps": [
+      {
+        "adaptor": "common",
+        "expression": "fn((state) => { state.result = dateFns.parse(state.data, 'MM/dd/yyyy', new Date()).toLocaleString(); return state; })"
+      }
+    ]
+  }
+}

integration-tests/cli/test/fixtures/invalid-start.json

@@ -5,7 +5,11 @@
   "workflow": {
     "steps": [
       {
-        "id": "x",
+        "id": "x1",
+        "expression": "fn((state) => state)"
+      },
+      {
+        "id": "x2",
         "expression": "fn((state) => state)"
       }
     ]

integration-tests/worker/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @openfn/integration-tests-worker
 
+## 1.0.39
+
+### Patch Changes
+
+- Updated dependencies [7ddc5d8]
+- Updated dependencies [4deb5d4]
+  - @openfn/ws-worker@1.1.4
+  - @openfn/engine-multi@1.1.4
+  - @openfn/lightning-mock@2.0.4
+
 ## 1.0.38
 
 ### Patch Changes

integration-tests/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-worker",
   "private": true,
-  "version": "1.0.38",
+  "version": "1.0.39",
   "description": "Lightning WOrker integration tests",
   "author": "Open Function Group <[email protected]>",
   "license": "ISC",

packages/cli/CHANGELOG.md

@@ -1,5 +1,23 @@
 # @openfn/cli
 
+## 1.2.0
+
+### Minor Changes
+
+- ea248a3: Allow step output to be cached
+  Accept fuzzy step ids in `--start`, `--end` and `--only`
+
+### Patch Changes
+
+- 7ddc5d8: Support expressions in lazy state operators
+- 4deb5d4: Recognise import aliases in job code
+- Updated dependencies [cecdb60]
+- Updated dependencies [4deb5d4]
+- Updated dependencies [7ddc5d8]
+  - @openfn/runtime@1.1.2
+  - @openfn/describe-package@0.0.19
+  - @openfn/compiler@0.1.2
+
 ## 1.1.4
 
 ### Patch Changes

packages/cli/README.md

@@ -127,6 +127,46 @@ If no command is specified, execute will run.
 
 To get more information about a command, including usage examples, run `openfn <command> help`, ie, `openfn compile help`.
 
+## Caching step output
+
+The CLI can write the output of every single step to disk (rather than just the final output). To do this, just run a job with the `--cache-steps` flag.
+
+```
+openfn tmp/job.js --cache-steps
+```
+
+The cached output is written to `.cli-cache/<workflow-name>/<step-name>.json`, relative to the input job or workflow file. This folder has a `.gitignore` file and should be ignored from your version control.
+
+So for a workflow at `./tmp/workflow.json` you'll get a cache path something like `./tmp/.cli-cache/workflow/step-1.json.`
+
+The cache is cleared when execution starts, so you know all artefacts in the cache folder relate to the last run.
+
+Step caching is disabled by default, but you can switch it on by setting the `OPENFN_ALWAYS_CACHE_STEPS` env var to `true`. To disable for a single execution, pass the `--no-compile-steps` flag.
+
+## Starting from a custom step
+
+When executing a workflow, the CLI will run from the first step (which is usually the first step in the `steps` array, unless `options.start` is set in the workflow).
+
+You can run from any starting step by passing `--start <step-name>`, like this:
+
+```
+openfn tmp/job.js --start upload-to-salesforce
+```
+
+If you previously cached the steps from this workflow, the CLI will automatically load the correct input state from the cache. Otherwise, you can pass in whatever state you need with `-s ./transformed-state.json`.
+
+You can also pass `--end` to make the workflow end early, or `--only` to only run a single step.
+
+All step names name supports "fuzzy" inputs. If you pass an exact step id, that step will always be the starting step. But you can also pass part of step name or id.
+
+So to match a step called with id `236baf56-e6c7-40f2-80ad-00d5a10b6b64` (such as you might download from Lightning), you can do:
+
+```
+openfn tmp/job.js --start 236b
+```
+
+Any unique continuous sequence of characters on the name or id will match. If there are multiple matches, an error will be thrown.
+
 ## Deploying Workflows
 
 > ⚠️ This feature is still in active development. Expect breaking changes.

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/cli/src/execute/command.ts

@@ -9,16 +9,20 @@ export type ExecuteOptions = Required<
     Opts,
     | 'adaptors'
     | 'autoinstall'
+    | 'baseDir'
+    | 'cacheSteps'
     | 'command'
     | 'compile'
     | 'expandAdaptors'
+    | 'end'
     | 'immutable'
     | 'ignoreImports'
     | 'expressionPath'
     | 'log'
     | 'logJson'
     | 'outputPath'
     | 'outputStdout'
+    | 'only'
     | 'path'
     | 'repoDir'
     | 'skipAdaptorValidation'
@@ -38,6 +42,7 @@ const options = [
 
   o.adaptors,
   o.autoinstall,
+  o.cacheSteps,
   o.compile,
   o.immutable,
   o.ignoreImports,

packages/cli/src/execute/execute.ts

@@ -1,9 +1,15 @@
-import run, { getNameAndVersion } from '@openfn/runtime';
+import run, { NOTIFY_JOB_COMPLETE, getNameAndVersion } from '@openfn/runtime';
 import type { ExecutionPlan, Job } from '@openfn/lexicon';
-import type { ModuleInfo, ModuleInfoMap } from '@openfn/runtime';
+import type {
+  ModuleInfo,
+  ModuleInfoMap,
+  NotifyJobCompletePayload,
+} from '@openfn/runtime';
 
-import createLogger, { RUNTIME, JOB } from '../util/logger';
-import { ExecuteOptions } from './command';
+import createLogger, { RUNTIME, JOB, Logger } from '../util/logger';
+import { saveToCache } from '../util/cache';
+
+import type { ExecuteOptions } from './command';
 
 type ExtendedModuleInfo = ModuleInfo & {
   name: string;
@@ -12,7 +18,8 @@ type ExtendedModuleInfo = ModuleInfo & {
 export default async (
   plan: ExecutionPlan,
   input: any,
-  opts: ExecuteOptions
+  opts: ExecuteOptions,
+  logger: Logger
 ): Promise<any> => {
   try {
     const result = await run(plan, input, {
@@ -23,6 +30,14 @@ export default async (
         repo: opts.repoDir,
         modules: parseAdaptors(plan),
       },
+      callbacks: {
+        notify: async (eventName, payload) => {
+          if (eventName === NOTIFY_JOB_COMPLETE) {
+            const { state, jobId } = payload as NotifyJobCompletePayload;
+            await saveToCache(plan, jobId, state, opts, logger);
+          }
+        },
+      },
     });
     return result;
   } catch (e: any) {