Backend renaming (1.0 version bumps plus the lexicon) (#585)

* lexicon: start building a central lexicon of definitions

* runtime: huge refactor of runtime core API

* runtime: more refactoring

* runtime: take initial state out of the execution plan

* fix tests

* runtime: changeset

* runtime: extra type tweakings

* runtime: readme

* runtime: jobs -> steps (mostly)

there are cases where job is more accurate and useful

* cli: start refactoring towrads new runtime API

Done a big chunk of execute but still a way to go

* cli: basically get the CLI working again

* cli: types

* cli: fix a bunch of tests, update workflow parsing

* cli: fix execute and compile tests

* cli: more test fixes

* fix more cli tests

* cli: fix integration tests

* cli: tidy

* runtime: remove strict mode

* remove strict mode

* cli: default workflow name to the file name

* runtime: tweak log output

* cli: remove log

* cli: types

* docs

* deploy: adjust logging

* engine: update types

* engine: update names and types

This is 90% of the basic rename done. Tests may even pass

* runtime: male statePropsToRemove a system options, rather than workflow specific

If a workflow wants to remove props, it'll add an fn bock

* engine: restore statePropsToRemove tests

* mock: update to lexicon

* worker: start mapping to lexicon. Handled run-> plan conversion

* worker: typings

* worker: fix all tests

* engine: types

* worker: fix cheeky test

somehow missed it last time

* tests: fix cli tests

* worker: update test

* package lock

* tests: update test

* changesets and housekeeping

* more housekeeping

* engine: tweak test

* runtime: tweak error messages

* worker: stricter type checkign on tests

* fix test

* typing in worker tests

* worker: update channel mock

* lexicon: docs

* Run -> LightningPlan

* version bumps for logger and mock

* Send worker versions (#593)

* worker: send worker and API versions to Lightning

* lexicon: fix API_VERSION export

.changeset/grumpy-pillows-tie.md

@@ -0,0 +1,12 @@
+---
+'@openfn/cli': major
+---
+
+The 1.0 Release of the CLI updates the language and input of the CLI to match the nomenclature of Lightning.
+
+See the readme for details of the new terminology.
+
+- Add support for execution plans
+- Deprecate old workflow format (old workflows are supported and will be automatically converted into the new "execution plans")
+- Update terminology across the codebase and docs
+- Remove strict mode

.changeset/old-planes-sort.md

@@ -0,0 +1,5 @@
+---
+'@openfn/deploy': patch
+---
+
+Log the result to success (not always)

.changeset/pretty-spoons-beam.md

@@ -0,0 +1,9 @@
+---
+'@openfn/runtime': major
+---
+
+The 1.0 release of the runtime updates the signatures and language of the runtime to match Lightning. It also includes some housekeeping.
+
+- Update main run() signature
+- Remove strict mode options
+- Integrate with lexicon

.changeset/ten-dingos-explode.md

@@ -0,0 +1,6 @@
+---
+'@openfn/engine-multi': major
+'@openfn/ws-worker': major
+---
+
+The 1.0 release updates the language and input of the CLI to match the nomenclature of Lightning.

.changeset/two-pugs-burn.md

@@ -0,0 +1,6 @@
+---
+'@openfn/lightning-mock': major
+'@openfn/logger': major
+---
+
+Symbolic 1.0 version release

integration-tests/cli/test/cli.test.ts

@@ -14,7 +14,7 @@ test.serial('openfn version', async (t) => {
 test.serial('openfn test', async (t) => {
   const { stdout } = await run(t.title);
   t.regex(stdout, /Versions:/);
-  t.regex(stdout, /Running test job.../);
+  t.regex(stdout, /Running test workflow/);
   t.regex(stdout, /Result: 42/);
 });
 

integration-tests/cli/test/errors.test.ts

@@ -2,19 +2,20 @@ import test from 'ava';
 import path from 'node:path';
 import run from '../src/run';
 import { extractLogs, assertLog } from '../src/util';
+import { stderr } from 'node:process';
 
 const jobsPath = path.resolve('test/fixtures');
 
 // These are all errors that will stop the CLI from even running
 
-test.serial('job not found', async (t) => {
+test.serial('expression not found', async (t) => {
   const { stdout, err } = await run('openfn blah.js --log-json');
   t.is(err.code, 1);
 
   const stdlogs = extractLogs(stdout);
 
-  assertLog(t, stdlogs, /job not found/i);
-  assertLog(t, stdlogs, /failed to load the job from blah.js/i);
+  assertLog(t, stdlogs, /expression not found/i);
+  assertLog(t, stdlogs, /failed to load the expression from blah.js/i);
   assertLog(t, stdlogs, /critical error: aborting command/i);
 });
 

integration-tests/cli/test/execute-workflow.test.ts

@@ -83,6 +83,15 @@ test.serial(
   }
 );
 
+// Run a new-style execution plan with custom start
+test.serial(`openfn ${jobsPath}/plan.json -i`, async (t) => {
+  const { err } = await run(t.title);
+  t.falsy(err);
+
+  const out = getJSON();
+  t.deepEqual(out.data.userId, 1);
+});
+
 test.serial(`openfn ${jobsPath}/wf-conditional.json`, async (t) => {
   const { err } = await run(t.title);
   t.falsy(err);
@@ -124,36 +133,6 @@ test.serial(
   }
 );
 
-test.serial(`openfn ${jobsPath}/wf-strict.json --strict`, async (t) => {
-  const { err } = await run(t.title);
-  t.falsy(err);
-
-  const out = getJSON();
-  t.deepEqual(out, {
-    data: {
-      name: 'jane',
-    },
-  });
-});
-
-test.serial(`openfn ${jobsPath}/wf-strict.json --no-strict`, async (t) => {
-  const { err } = await run(t.title);
-  t.falsy(err);
-
-  const out = getJSON();
-  t.deepEqual(out, {
-    x: 22,
-    data: {
-      name: 'jane',
-    },
-    references: [
-      {
-        name: 'bob',
-      },
-    ],
-  });
-});
-
 test.serial(
   `openfn ${jobsPath}/wf-errors.json -S "{ \\"data\\": { \\"number\\": 2 } }"`,
   async (t) => {
@@ -169,8 +148,8 @@ test.serial(
   }
 );
 
-test.serial(
-  `openfn ${jobsPath}/wf-errors.json -S "{ \\"data\\": { \\"number\\": 32 } }"`,
+test.serial.only(
+  `openfn ${jobsPath}/wf-errors.json -iS "{ \\"data\\": { \\"number\\": 32 } }"`,
   async (t) => {
     const { err } = await run(t.title);
     t.falsy(err);
@@ -189,7 +168,7 @@ test.serial(
             severity: 'fail',
             source: 'runtime',
           },
-          jobId: 'start',
+          stepId: 'start',
           message: 'abort',
           type: 'JobError',
         },

integration-tests/cli/test/fixtures/circular.json

@@ -1,14 +1,16 @@
 {
-  "jobs": [
-    {
-      "id": "a",
-      "expression": "x",
-      "next": { "b": true }
-    },
-    {
-      "id": "b",
-      "expression": "x",
-      "next": { "a": true }
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "id": "a",
+        "expression": "x",
+        "next": { "b": true }
+      },
+      {
+        "id": "b",
+        "expression": "x",
+        "next": { "a": true }
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/invalid-config-path.json

@@ -1,7 +1,10 @@
 {
-  "jobs": [
-    {
-      "configuration": "does-not-exist.json"
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "configuration": "does-not-exist.json",
+        "expression": "."
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/invalid-exp-path.json

@@ -1,7 +1,9 @@
 {
-  "jobs": [
-    {
-      "expression": "does-not-exist.js"
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "expression": "does-not-exist.js"
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/invalid-start.json

@@ -1,9 +1,13 @@
 {
-  "start": "nope",
-  "jobs": [
-    {
-      "id": "x",
-      "expression": "fn((state) => state)"
-    }
-  ]
+  "options": {
+    "start": "nope"
+  },
+  "workflow": {
+    "steps": [
+      {
+        "id": "x",
+        "expression": "fn((state) => state)"
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/invalid-syntax.json

@@ -1,7 +1,9 @@
 {
-  "jobs": [
-    {
-      "expression": "invalid.js"
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "expression": "invalid.js"
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/multiple-inputs.json

@@ -1,18 +1,20 @@
 {
-  "jobs": [
-    {
-      "id": "a",
-      "expression": "x",
-      "next": { "b": true, "c": true }
-    },
-    {
-      "id": "b",
-      "expression": "x",
-      "next": { "c": true }
-    },
-    {
-      "id": "c",
-      "expression": "x"
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "id": "a",
+        "expression": "x",
+        "next": { "b": true, "c": true }
+      },
+      {
+        "id": "b",
+        "expression": "x",
+        "next": { "c": true }
+      },
+      {
+        "id": "c",
+        "expression": "x"
+      }
+    ]
+  }
 }

integration-tests/cli/test/fixtures/plan.json

@@ -0,0 +1,19 @@
+{
+  "options": {
+    "start": "b"
+  },
+  "workflow": {
+    "steps": [
+      {
+        "id": "a",
+        "adaptor": "common",
+        "expression": "fn((state) => { return state; });"
+      },
+      {
+        "id": "b",
+        "adaptor": "http",
+        "expression": "get('https://jsonplaceholder.typicode.com/todos/1')"
+      }
+    ]
+  }
+}

integration-tests/cli/test/fixtures/wf-array.json

@@ -1,21 +1,23 @@
 {
-  "jobs": [
-    {
-      "id": "a",
-      "adaptor": "common",
-      "expression": "fn((state) => { if (!state.data.items) { state.data.items = []; } return state; });",
-      "next": { "b": true }
-    },
-    {
-      "id": "b",
-      "adaptor": "common",
-      "expression": "fn((state) => { state.data.items.push('b'); return state; });",
-      "next": { "c": true }
-    },
-    {
-      "id": "c",
-      "adaptor": "common",
-      "expression": "fn((state) => { state.data.items.push('c'); return state; });"
-    }
-  ]
+  "workflow": {
+    "steps": [
+      {
+        "id": "a",
+        "adaptor": "common",
+        "expression": "fn((state) => { if (!state.data.items) { state.data.items = []; } return state; });",
+        "next": { "b": true }
+      },
+      {
+        "id": "b",
+        "adaptor": "common",
+        "expression": "fn((state) => { state.data.items.push('b'); return state; });",
+        "next": { "c": true }
+      },
+      {
+        "id": "c",
+        "adaptor": "common",
+        "expression": "fn((state) => { state.data.items.push('c'); return state; });"
+      }
+    ]
+  }
 }