diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 96dccc599..93f047634 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/cli +## 1.8.1 + +### Patch Changes + +- Updated dependencies [0d53f9b] + - @openfn/deploy@0.7.0 + ## 1.8.0 ### Minor Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index dd8498585..7043d9383 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.8.0", + "version": "1.8.1", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index a5b4aaab7..ff172b7ea 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/deploy +## 0.7.0 + +### Minor Changes + +- 0d53f9b: Add support for basic project-credential management (add, associate with jobs) via the CLI + ## 0.6.0 ### Minor Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index c780b5262..ba962c003 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.6.0", + "version": "0.7.0", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/packages/deploy/src/pull.ts b/packages/deploy/src/pull.ts index 4df7b7f11..58ee3cb31 100644 --- a/packages/deploy/src/pull.ts +++ b/packages/deploy/src/pull.ts @@ -25,6 +25,7 @@ async function getAllSpecJobs( name: specJob.name, adaptor: specJob.adaptor, body: specJob.body, + credential: specJob.credential, }); } } diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index d1b11b936..9f609ae8a 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import { deepClone } from 'fast-json-patch'; import { + CredentialState, ProjectPayload, ProjectSpec, ProjectState, @@ -29,7 +30,22 @@ function stringifyJobBody(body: SpecJobBody): string { } } +function getStateJobCredential( + specJobCredential: string, + stateCredentials: ProjectState['project_credentials'] +): string { + if (!stateCredentials[specJobCredential]) { + throw new DeployError( + `Could not find a credential with name: ${specJobCredential}`, + 'VALIDATION_ERROR' + ); + } + + return stateCredentials[specJobCredential].id; +} + function mergeJobs( + credentials: ProjectState['project_credentials'], stateJobs: WorkflowState['jobs'], specJobs: WorkflowSpec['jobs'] ): WorkflowState['jobs'] { @@ -43,6 +59,9 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && + getStateJobCredential(specJob.credential, credentials), }, ]; } @@ -59,6 +78,9 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && + getStateJobCredential(specJob.credential, credentials), }, ]; } @@ -195,10 +217,46 @@ export function mergeSpecIntoState( spec: ProjectSpec, logger?: Logger ): ProjectState { + const nextCredentials = Object.fromEntries( + splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( + ([credentialKey, stateCredential, specCredential]) => { + if (specCredential && !stateCredential) { + return [ + credentialKey, + { + id: crypto.randomUUID(), + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + if (specCredential && stateCredential) { + return [ + credentialKey, + { + id: stateCredential.id, + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + throw new DeployError( + `Invalid credential spec or corrupted state for credential: ${ + stateCredential?.name || specCredential?.name + } (${stateCredential?.owner || specCredential?.owner})`, + 'VALIDATION_ERROR' + ); + } + ) + ); + const nextWorkflows = Object.fromEntries( splitZip(oldState.workflows, spec.workflows).map( ([workflowKey, stateWorkflow, specWorkflow]) => { const nextJobs = mergeJobs( + nextCredentials, stateWorkflow?.jobs || {}, specWorkflow?.jobs || {} ); @@ -258,6 +316,7 @@ export function mergeSpecIntoState( id: oldState.id || crypto.randomUUID(), name: spec.name, workflows: nextWorkflows, + project_credentials: nextCredentials, }; if (spec.description) projectState.description = spec.description; @@ -295,8 +354,18 @@ export function getStateFromProjectPayload( return stateWorkflow as WorkflowState; }); + const project_credentials = (project.project_credentials || []).reduce( + (acc, credential) => { + const key = hyphenate(`${credential.owner} ${credential.name}`); + acc[key] = credential; + return acc; + }, + {} as Record + ); + return { ...project, + project_credentials, workflows, }; } @@ -347,8 +416,18 @@ export function mergeProjectPayloadIntoState( ) ); + const nextCredentials = Object.fromEntries( + idKeyPairs( + project.project_credentials || {}, + state.project_credentials || {} + ).map(([key, nextCredential, _state]) => { + return [key, nextCredential]; + }) + ); + return { ...project, + project_credentials: nextCredentials, workflows: nextWorkflows, }; } @@ -385,8 +464,12 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { }; }); + const project_credentials: ProjectPayload['project_credentials'] = + Object.values(state.project_credentials); + return { ...state, + project_credentials, workflows, }; } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index a4ea7f52d..d721541e3 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -1,8 +1,9 @@ export type StateJob = { - id?: string; + id: string; name: string; adaptor: string; body: string; + project_credential_id: string | null; delete?: boolean; }; @@ -18,6 +19,7 @@ export type SpecJob = { name: string; adaptor: string; body: SpecJobBody; + credential: string | null; }; export type Trigger = { @@ -57,10 +59,22 @@ export type WorkflowSpec = { edges?: Record; }; +export type CredentialSpec = { + name: string; + owner: string; +}; + +export type CredentialState = { + id: string; + name: string; + owner: string; +}; + export interface ProjectSpec { name: string; description: string; workflows: Record; + credentials: Record; } export interface WorkflowState { @@ -82,12 +96,14 @@ export interface ProjectState { name: string; description: string; workflows: Record; + project_credentials: Record; } export interface ProjectPayload { id: string; name: string; description: string; + project_credentials: Concrete[]; workflows: { id: string; name: string; diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index 79e218ab2..b575c92e1 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -117,6 +117,12 @@ export async function parseAndValidate( } } + if (pair.key && pair.key.value === 'credentials') { + if (pair.value.value === null) { + return doc.createPair('credentials', {}); + } + } + if (pair.key && pair.key.value === 'jobs') { if (pair.value.value === null) { errors.push({ diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index c97ed0182..fbff57a2e 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -4,6 +4,7 @@ export function fullExampleSpec() { return { name: 'my project', description: 'some helpful description', + credentials: {}, workflows: { 'workflow-one': { name: 'workflow one', @@ -15,11 +16,13 @@ export function fullExampleSpec() { path: 'somefile.js', content: '', }, + credential: null, }, 'job-b': { name: 'job b', adaptor: '@openfn/language-common@latest', body: '', + credential: null, }, }, triggers: { @@ -53,6 +56,7 @@ export function fullExampleState() { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'some helpful description', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -63,12 +67,14 @@ export function fullExampleState() { name: 'job a', adaptor: '@openfn/language-common@latest', body: '', + project_credential_id: null, }, 'job-b': { id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37', name: 'job b', adaptor: '@openfn/language-common@latest', body: '', + project_credential_id: null, }, }, triggers: { @@ -110,6 +116,13 @@ export const lightningProjectPayload = { updated_at: '2023-08-25T08:57:31', scheduled_deletion: null, requires_mfa: false, + project_credentials: [ + { + id: '25f48989-d349-4eb8-99c3-923ebba5b116', + name: 'Basic Auth', + owner: 'email@test.com', + }, + ], workflows: [ { id: '05fab294-98dc-4d7d-85f3-024b2b0e6897', @@ -154,24 +167,28 @@ export const lightningProjectPayload = { name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116', }, { id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, { id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, { id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, ], triggers: [ @@ -223,6 +240,13 @@ export const lightningProjectState = { updated_at: '2023-08-25T08:57:31', scheduled_deletion: null, requires_mfa: false, + project_credentials: { + 'email@test.com-Basic-Auth': { + id: '25f48989-d349-4eb8-99c3-923ebba5b116', + name: 'Basic Auth', + owner: 'email@test.com', + }, + }, workflows: { 'OpenHIE-Workflow': { id: '05fab294-98dc-4d7d-85f3-024b2b0e6897', @@ -267,24 +291,28 @@ export const lightningProjectState = { name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116', }, 'Send-to-OpenHIM-to-route-to-SHR': { id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, 'Notify-CHW-upload-successful': { id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, 'Notify-CHW-upload-failed': { id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, }, triggers: { diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index 67bdec761..760813257 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -71,6 +71,7 @@ test('toNextState adding a job', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', + project_credential_id: undefined, }, }, triggers: { @@ -88,6 +89,7 @@ test('toNextState adding a job', (t) => { id: 'ecb683d1-5e5a-4c4f-9165-e143e2eeeb48', name: 'project-name', description: 'my test project', + project_credentials: {}, }); }); @@ -114,6 +116,7 @@ test('toNextState with empty state', (t) => { id: result.id, name: 'my project', description: 'some helpful description', + project_credentials: {}, workflows: { 'workflow-one': { id: jp.query(result, '$..workflows["workflow-one"].id')[0], @@ -124,12 +127,14 @@ test('toNextState with empty state', (t) => { adaptor: '@openfn/language-common@latest', name: 'job a', body: '', + project_credential_id: null, }, 'job-b': { id: getItem(result, 'jobs', 'job-b').id, adaptor: '@openfn/language-common@latest', name: 'job b', body: '', + project_credential_id: null, }, }, triggers: { @@ -170,6 +175,7 @@ test('toNextState with no changes', (t) => { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'for the humans', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -180,6 +186,7 @@ test('toNextState with no changes', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', + project_credential_id: undefined, }, }, triggers: { @@ -283,6 +290,7 @@ test('toNextState with a new job', (t) => { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'some other description', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -293,12 +301,14 @@ test('toNextState with a new job', (t) => { name: 'job a', body: 'foo()', adaptor: '@openfn/language-adaptor', + project_credential_id: undefined, }, 'job-b': { id: getItem(result, 'jobs', 'job-b').id, name: 'job b', adaptor: undefined, body: undefined, + project_credential_id: undefined, }, }, triggers: { @@ -448,6 +458,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { t.deepEqual(state, { id: 'xyz', name: 'project', + project_credentials: {}, workflows: { a: { id: 'wf-a', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af598fe53..b6f2b5bc3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -468,8 +468,6 @@ importers: specifier: ^5.1.6 version: 5.1.6 - packages/engine-multi/tmp/a/b/c: {} - packages/engine-multi/tmp/repo: {} packages/lexicon: