From 22cd1c3c9c8d2c84157d30733b3e0665629da6a5 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Tue, 13 Aug 2024 11:57:54 +0300 Subject: [PATCH 1/7] support credentials --- packages/deploy/src/index.ts | 2 +- packages/deploy/src/stateTransform.ts | 72 +++++++++++++++++++++++++++ packages/deploy/src/types.ts | 19 ++++++- packages/deploy/src/validator.ts | 6 +++ 4 files changed, 96 insertions(+), 3 deletions(-) diff --git a/packages/deploy/src/index.ts b/packages/deploy/src/index.ts index 1c22fbd97..f56891a64 100644 --- a/packages/deploy/src/index.ts +++ b/packages/deploy/src/index.ts @@ -76,7 +76,7 @@ export async function getState(path: string) { return await readState(path); } catch (error: any) { if (error.code === 'ENOENT') { - return { workflows: {} } as ProjectState; + return { workflows: {}, project_credentials: {} } as ProjectState; } else { throw error; } diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index d1b11b936..6f5d20a65 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -1,6 +1,9 @@ +// @ts-nocheck +// @ts-ignore import crypto from 'crypto'; import { deepClone } from 'fast-json-patch'; import { + CredentialState, ProjectPayload, ProjectSpec, ProjectState, @@ -30,6 +33,7 @@ function stringifyJobBody(body: SpecJobBody): string { } function mergeJobs( + credentials: ProjectState['project_credentials'], stateJobs: WorkflowState['jobs'], specJobs: WorkflowSpec['jobs'] ): WorkflowState['jobs'] { @@ -43,6 +47,8 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && credentials[specJob.credential]?.id, }, ]; } @@ -59,6 +65,8 @@ function mergeJobs( name: specJob.name, adaptor: specJob.adaptor, body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && credentials[specJob.credential]?.id, }, ]; } @@ -195,10 +203,48 @@ export function mergeSpecIntoState( spec: ProjectSpec, logger?: Logger ): ProjectState { + const nextCredentials = Object.fromEntries( + splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( + ([credentialKey, stateCredential, specCredential]) => { + if (specCredential && !stateCredential) { + return [ + credentialKey, + { + id: crypto.randomUUID(), + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + if (specCredential && stateCredential) { + return [ + credentialKey, + { + id: stateCredential.id, + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + if (!specCredential && !isEmpty(stateCredential || {})) { + logger?.error('Critical error! Cannot continue'); + logger?.error( + 'Crdential found in project state but not spec:', + `${stateCredential.name} (${stateCredential.owner})` + ); + process.exit(1); + } + } + ) + ); + const nextWorkflows = Object.fromEntries( splitZip(oldState.workflows, spec.workflows).map( ([workflowKey, stateWorkflow, specWorkflow]) => { const nextJobs = mergeJobs( + nextCredentials, stateWorkflow?.jobs || {}, specWorkflow?.jobs || {} ); @@ -258,6 +304,7 @@ export function mergeSpecIntoState( id: oldState.id || crypto.randomUUID(), name: spec.name, workflows: nextWorkflows, + project_credentials: nextCredentials, }; if (spec.description) projectState.description = spec.description; @@ -265,6 +312,8 @@ export function mergeSpecIntoState( return projectState as ProjectState; } +// @ts-nocheck +// @ts-ignore export function getStateFromProjectPayload( project: ProjectPayload ): ProjectState { @@ -295,8 +344,18 @@ export function getStateFromProjectPayload( return stateWorkflow as WorkflowState; }); + const project_credentials = project.project_credentials.reduce( + (acc, credential) => { + const key = hyphenate(`${credential.owner} ${credential.name}`); + acc[key] = credential; + return acc; + }, + {} as Record + ); + return { ...project, + project_credentials, workflows, }; } @@ -347,8 +406,17 @@ export function mergeProjectPayloadIntoState( ) ); + const nextCredentials = Object.fromEntries( + idKeyPairs(project.project_credentials, state.project_credentials).map( + ([key, nextCredential, _state]) => { + return [key, nextCredential]; + } + ) + ); + return { ...project, + project_credentials: nextCredentials, workflows: nextWorkflows, }; } @@ -385,8 +453,12 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { }; }); + const project_credentials: ProjectPayload['project_credentials'] = + Object.values(state.project_credentials); + return { ...state, + project_credentials, workflows, }; } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index a4ea7f52d..37d6bceae 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -1,8 +1,9 @@ export type StateJob = { - id?: string; + id: string; name: string; adaptor: string; body: string; + project_credential_id: string | null; delete?: boolean; }; @@ -14,10 +15,10 @@ export type SpecJobBody = }; export type SpecJob = { - id?: string; name: string; adaptor: string; body: SpecJobBody; + credential: string | null; }; export type Trigger = { @@ -57,10 +58,22 @@ export type WorkflowSpec = { edges?: Record; }; +export type CredentialSpec = { + name: string; + owner: string; +}; + +export type CredentialState = { + id: string; + name: string; + owner: string; +}; + export interface ProjectSpec { name: string; description: string; workflows: Record; + credentials: Record; } export interface WorkflowState { @@ -82,12 +95,14 @@ export interface ProjectState { name: string; description: string; workflows: Record; + project_credentials: Record; } export interface ProjectPayload { id: string; name: string; description: string; + project_credentials: Concrete[]; workflows: { id: string; name: string; diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index 79e218ab2..b575c92e1 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -117,6 +117,12 @@ export async function parseAndValidate( } } + if (pair.key && pair.key.value === 'credentials') { + if (pair.value.value === null) { + return doc.createPair('credentials', {}); + } + } + if (pair.key && pair.key.value === 'jobs') { if (pair.value.value === null) { errors.push({ From 2df24fb5396d4f6b66ed865d64b8061c7a9dc6e2 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Tue, 13 Aug 2024 17:23:52 +0300 Subject: [PATCH 2/7] fix failing tests --- packages/deploy/src/index.ts | 2 +- packages/deploy/src/stateTransform.ts | 31 +++++++++------------ packages/deploy/test/fixtures.ts | 28 +++++++++++++++++++ packages/deploy/test/stateTransform.test.ts | 11 ++++++++ 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/packages/deploy/src/index.ts b/packages/deploy/src/index.ts index f56891a64..1c22fbd97 100644 --- a/packages/deploy/src/index.ts +++ b/packages/deploy/src/index.ts @@ -76,7 +76,7 @@ export async function getState(path: string) { return await readState(path); } catch (error: any) { if (error.code === 'ENOENT') { - return { workflows: {}, project_credentials: {} } as ProjectState; + return { workflows: {} } as ProjectState; } else { throw error; } diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index 6f5d20a65..3241611c1 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -1,5 +1,3 @@ -// @ts-nocheck -// @ts-ignore import crypto from 'crypto'; import { deepClone } from 'fast-json-patch'; import { @@ -228,14 +226,12 @@ export function mergeSpecIntoState( ]; } - if (!specCredential && !isEmpty(stateCredential || {})) { - logger?.error('Critical error! Cannot continue'); - logger?.error( - 'Crdential found in project state but not spec:', - `${stateCredential.name} (${stateCredential.owner})` - ); - process.exit(1); - } + throw new DeployError( + `Invalid credential spec or corrupted state for credential: ${ + stateCredential?.name || specCredential?.name + } (${stateCredential?.owner || specCredential?.owner})`, + 'VALIDATION_ERROR' + ); } ) ); @@ -312,8 +308,6 @@ export function mergeSpecIntoState( return projectState as ProjectState; } -// @ts-nocheck -// @ts-ignore export function getStateFromProjectPayload( project: ProjectPayload ): ProjectState { @@ -344,7 +338,7 @@ export function getStateFromProjectPayload( return stateWorkflow as WorkflowState; }); - const project_credentials = project.project_credentials.reduce( + const project_credentials = (project.project_credentials || []).reduce( (acc, credential) => { const key = hyphenate(`${credential.owner} ${credential.name}`); acc[key] = credential; @@ -407,11 +401,12 @@ export function mergeProjectPayloadIntoState( ); const nextCredentials = Object.fromEntries( - idKeyPairs(project.project_credentials, state.project_credentials).map( - ([key, nextCredential, _state]) => { - return [key, nextCredential]; - } - ) + idKeyPairs( + project.project_credentials || {}, + state.project_credentials || {} + ).map(([key, nextCredential, _state]) => { + return [key, nextCredential]; + }) ); return { diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index c97ed0182..fbff57a2e 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -4,6 +4,7 @@ export function fullExampleSpec() { return { name: 'my project', description: 'some helpful description', + credentials: {}, workflows: { 'workflow-one': { name: 'workflow one', @@ -15,11 +16,13 @@ export function fullExampleSpec() { path: 'somefile.js', content: '', }, + credential: null, }, 'job-b': { name: 'job b', adaptor: '@openfn/language-common@latest', body: '', + credential: null, }, }, triggers: { @@ -53,6 +56,7 @@ export function fullExampleState() { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'some helpful description', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -63,12 +67,14 @@ export function fullExampleState() { name: 'job a', adaptor: '@openfn/language-common@latest', body: '', + project_credential_id: null, }, 'job-b': { id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37', name: 'job b', adaptor: '@openfn/language-common@latest', body: '', + project_credential_id: null, }, }, triggers: { @@ -110,6 +116,13 @@ export const lightningProjectPayload = { updated_at: '2023-08-25T08:57:31', scheduled_deletion: null, requires_mfa: false, + project_credentials: [ + { + id: '25f48989-d349-4eb8-99c3-923ebba5b116', + name: 'Basic Auth', + owner: 'email@test.com', + }, + ], workflows: [ { id: '05fab294-98dc-4d7d-85f3-024b2b0e6897', @@ -154,24 +167,28 @@ export const lightningProjectPayload = { name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116', }, { id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, { id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, { id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, ], triggers: [ @@ -223,6 +240,13 @@ export const lightningProjectState = { updated_at: '2023-08-25T08:57:31', scheduled_deletion: null, requires_mfa: false, + project_credentials: { + 'email@test.com-Basic-Auth': { + id: '25f48989-d349-4eb8-99c3-923ebba5b116', + name: 'Basic Auth', + owner: 'email@test.com', + }, + }, workflows: { 'OpenHIE-Workflow': { id: '05fab294-98dc-4d7d-85f3-024b2b0e6897', @@ -267,24 +291,28 @@ export const lightningProjectState = { name: 'FHIR standard Data with change', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116', }, 'Send-to-OpenHIM-to-route-to-SHR': { id: 'ed3f110a-c800-479b-9576-47bb87e9ad57', name: 'Send to OpenHIM to route to SHR', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, 'Notify-CHW-upload-successful': { id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b', name: 'Notify CHW upload successful', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, 'Notify-CHW-upload-failed': { id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac', name: 'Notify CHW upload failed', body: 'fn(state => state);\n', adaptor: '@openfn/language-http@latest', + project_credential_id: null, }, }, triggers: { diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index 67bdec761..760813257 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -71,6 +71,7 @@ test('toNextState adding a job', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', + project_credential_id: undefined, }, }, triggers: { @@ -88,6 +89,7 @@ test('toNextState adding a job', (t) => { id: 'ecb683d1-5e5a-4c4f-9165-e143e2eeeb48', name: 'project-name', description: 'my test project', + project_credentials: {}, }); }); @@ -114,6 +116,7 @@ test('toNextState with empty state', (t) => { id: result.id, name: 'my project', description: 'some helpful description', + project_credentials: {}, workflows: { 'workflow-one': { id: jp.query(result, '$..workflows["workflow-one"].id')[0], @@ -124,12 +127,14 @@ test('toNextState with empty state', (t) => { adaptor: '@openfn/language-common@latest', name: 'job a', body: '', + project_credential_id: null, }, 'job-b': { id: getItem(result, 'jobs', 'job-b').id, adaptor: '@openfn/language-common@latest', name: 'job b', body: '', + project_credential_id: null, }, }, triggers: { @@ -170,6 +175,7 @@ test('toNextState with no changes', (t) => { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'for the humans', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -180,6 +186,7 @@ test('toNextState with no changes', (t) => { name: 'new job', adaptor: '@openfn/language-adaptor', body: 'foo()', + project_credential_id: undefined, }, }, triggers: { @@ -283,6 +290,7 @@ test('toNextState with a new job', (t) => { id: 'be156ab1-8426-4151-9a18-4045142f9ec0', name: 'my project', description: 'some other description', + project_credentials: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -293,12 +301,14 @@ test('toNextState with a new job', (t) => { name: 'job a', body: 'foo()', adaptor: '@openfn/language-adaptor', + project_credential_id: undefined, }, 'job-b': { id: getItem(result, 'jobs', 'job-b').id, name: 'job b', adaptor: undefined, body: undefined, + project_credential_id: undefined, }, }, triggers: { @@ -448,6 +458,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { t.deepEqual(state, { id: 'xyz', name: 'project', + project_credentials: {}, workflows: { a: { id: 'wf-a', From 9eec3532d95f61401f78588ee4eae17c41314171 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Tue, 13 Aug 2024 17:53:46 +0300 Subject: [PATCH 3/7] throw an error incase the referenced credential doesnt exist --- packages/deploy/src/stateTransform.ts | 759 +++++++++++++------------- 1 file changed, 393 insertions(+), 366 deletions(-) diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index 3241611c1..0ba918cab 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -29,431 +29,458 @@ function stringifyJobBody(body: SpecJobBody): string { return body; } } - -function mergeJobs( - credentials: ProjectState['project_credentials'], - stateJobs: WorkflowState['jobs'], - specJobs: WorkflowSpec['jobs'] -): WorkflowState['jobs'] { - return Object.fromEntries( - splitZip(stateJobs, specJobs || {}).map(([jobKey, stateJob, specJob]) => { - if (specJob && !stateJob) { - return [ - jobKey, - { - id: crypto.randomUUID(), - name: specJob.name, - adaptor: specJob.adaptor, - body: stringifyJobBody(specJob.body), - project_credential_id: - specJob.credential && credentials[specJob.credential]?.id, - }, - ]; - } - - if (!specJob && stateJob) { - return [jobKey, { id: stateJob.id, delete: true }]; - } - - if (specJob && stateJob) { - return [ - jobKey, - { - id: stateJob.id, - name: specJob.name, - adaptor: specJob.adaptor, - body: stringifyJobBody(specJob.body), - project_credential_id: - specJob.credential && credentials[specJob.credential]?.id, - }, - ]; - } - + function getStateJobCredential( + specJobCredential: undefined | null | string, + stateCredentials: ProjectState['project_credentials'] + ): undefined | null | string { + if ( + specJobCredential && + typeof stateCredentials[specJobCredential] === 'undefined' + ) { throw new DeployError( - `Invalid job spec or corrupted state for job with key: ${String( - jobKey - )}`, + `Could not find a credential with name: ${specJobCredential}`, 'VALIDATION_ERROR' ); - }) - ); -} - -// Given two objects, find the value of a key in the first object, or the second -// object, falling back to a default value. -function pickValue( - first: Record, - second: Record, - key: string, - defaultValue: any -): any { - if (key in first) { - return first[key]; - } + } - if (key in second) { - return second[key]; + return specJobCredential && stateCredentials[specJobCredential].id; } - return defaultValue; -} - -function mergeTriggers( - stateTriggers: WorkflowState['triggers'], - specTriggers: WorkflowSpec['triggers'] -): WorkflowState['triggers'] { - return Object.fromEntries( - splitZip(stateTriggers, specTriggers!).map( - ([triggerKey, stateTrigger, specTrigger]) => { - if (specTrigger && !stateTrigger) { + function mergeJobs( + credentials: ProjectState['project_credentials'], + stateJobs: WorkflowState['jobs'], + specJobs: WorkflowSpec['jobs'] + ): WorkflowState['jobs'] { + return Object.fromEntries( + splitZip(stateJobs, specJobs || {}).map(([jobKey, stateJob, specJob]) => { + if (specJob && !stateJob) { return [ - triggerKey, + jobKey, { id: crypto.randomUUID(), - ...pickKeys(specTrigger, ['type', 'enabled']), - ...(specTrigger.type === 'cron' - ? { cron_expression: specTrigger.cron_expression } - : {}), + name: specJob.name, + adaptor: specJob.adaptor, + body: stringifyJobBody(specJob.body), + project_credential_id: getStateJobCredential( + specJob.credential, + credentials + ), }, ]; } - if (!specTrigger && stateTrigger) { - return [triggerKey, { id: stateTrigger!.id, delete: true }]; + if (!specJob && stateJob) { + return [jobKey, { id: stateJob.id, delete: true }]; } - // prefer spec, but use state if spec is missing, or default - return [ - triggerKey, - { - id: stateTrigger!.id, - ...{ - type: pickValue(specTrigger!, stateTrigger!, 'type', 'webhook'), - enabled: pickValue(specTrigger!, stateTrigger!, 'enabled', true), - ...(specTrigger!.type === 'cron' - ? { cron_expression: specTrigger!.cron_expression } - : {}), - }, - }, - ]; - } - ) - ); -} - -function mergeEdges( - { jobs, triggers }: Pick, - stateEdges: WorkflowState['edges'], - specEdges: WorkflowSpec['edges'] -): WorkflowState['edges'] { - return Object.fromEntries( - splitZip(stateEdges, specEdges || {}).map( - ([edgeKey, stateEdge, specEdge]) => { - // build a 'new edge', based off the spec and existing jobs and triggers - function convertToStateEdge( - jobs: WorkflowState['jobs'], - triggers: WorkflowState['triggers'], - specEdge: SpecEdge, - id: string - ): StateEdge { - const edge: StateEdge = assignIfTruthy( + if (specJob && stateJob) { + return [ + jobKey, { - id, - condition_type: specEdge.condition_type ?? null, - target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '', - enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true), + id: stateJob.id, + name: specJob.name, + adaptor: specJob.adaptor, + body: stringifyJobBody(specJob.body), + project_credential_id: getStateJobCredential( + specJob.credential, + credentials + ), }, - { - condition_type: specEdge.condition_type, - condition_expression: specEdge.condition_expression, - condition_label: specEdge.condition_label, - source_job_id: jobs[specEdge.source_job ?? -1]?.id, - source_trigger_id: triggers[specEdge.source_trigger ?? -1]?.id, - } - ); - - return edge; - } - - if (specEdge && !stateEdge) { - return [ - edgeKey, - convertToStateEdge(jobs, triggers, specEdge, crypto.randomUUID()), ]; } - if (!specEdge && stateEdge) { - return [edgeKey, { id: stateEdge.id, delete: true }]; - } + throw new DeployError( + `Invalid job spec or corrupted state for job with key: ${String( + jobKey + )}`, + 'VALIDATION_ERROR' + ); + }) + ); + } - return [ - edgeKey, - convertToStateEdge(jobs, triggers, specEdge!, stateEdge!.id), - ]; - } - ) - ); -} + // Given two objects, find the value of a key in the first object, or the second + // object, falling back to a default value. + function pickValue( + first: Record, + second: Record, + key: string, + defaultValue: any + ): any { + if (key in first) { + return first[key]; + } + + if (key in second) { + return second[key]; + } -// Prepare the next state, based on the current state and the spec. -export function mergeSpecIntoState( - oldState: ProjectState, - spec: ProjectSpec, - logger?: Logger -): ProjectState { - const nextCredentials = Object.fromEntries( - splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( - ([credentialKey, stateCredential, specCredential]) => { - if (specCredential && !stateCredential) { + return defaultValue; + } + + function mergeTriggers( + stateTriggers: WorkflowState['triggers'], + specTriggers: WorkflowSpec['triggers'] + ): WorkflowState['triggers'] { + return Object.fromEntries( + splitZip(stateTriggers, specTriggers!).map( + ([triggerKey, stateTrigger, specTrigger]) => { + if (specTrigger && !stateTrigger) { + return [ + triggerKey, + { + id: crypto.randomUUID(), + ...pickKeys(specTrigger, ['type', 'enabled']), + ...(specTrigger.type === 'cron' + ? { cron_expression: specTrigger.cron_expression } + : {}), + }, + ]; + } + + if (!specTrigger && stateTrigger) { + return [triggerKey, { id: stateTrigger!.id, delete: true }]; + } + + // prefer spec, but use state if spec is missing, or default return [ - credentialKey, + triggerKey, { - id: crypto.randomUUID(), - name: specCredential.name, - owner: specCredential.owner, + id: stateTrigger!.id, + ...{ + type: pickValue(specTrigger!, stateTrigger!, 'type', 'webhook'), + enabled: pickValue( + specTrigger!, + stateTrigger!, + 'enabled', + true + ), + ...(specTrigger!.type === 'cron' + ? { cron_expression: specTrigger!.cron_expression } + : {}), + }, }, ]; } + ) + ); + } + + function mergeEdges( + { jobs, triggers }: Pick, + stateEdges: WorkflowState['edges'], + specEdges: WorkflowSpec['edges'] + ): WorkflowState['edges'] { + return Object.fromEntries( + splitZip(stateEdges, specEdges || {}).map( + ([edgeKey, stateEdge, specEdge]) => { + // build a 'new edge', based off the spec and existing jobs and triggers + function convertToStateEdge( + jobs: WorkflowState['jobs'], + triggers: WorkflowState['triggers'], + specEdge: SpecEdge, + id: string + ): StateEdge { + const edge: StateEdge = assignIfTruthy( + { + id, + condition_type: specEdge.condition_type ?? null, + target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '', + enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true), + }, + { + condition_type: specEdge.condition_type, + condition_expression: specEdge.condition_expression, + condition_label: specEdge.condition_label, + source_job_id: jobs[specEdge.source_job ?? -1]?.id, + source_trigger_id: triggers[specEdge.source_trigger ?? -1]?.id, + } + ); + + return edge; + } + + if (specEdge && !stateEdge) { + return [ + edgeKey, + convertToStateEdge(jobs, triggers, specEdge, crypto.randomUUID()), + ]; + } + + if (!specEdge && stateEdge) { + return [edgeKey, { id: stateEdge.id, delete: true }]; + } - if (specCredential && stateCredential) { return [ - credentialKey, - { - id: stateCredential.id, - name: specCredential.name, - owner: specCredential.owner, - }, + edgeKey, + convertToStateEdge(jobs, triggers, specEdge!, stateEdge!.id), ]; } + ) + ); + } - throw new DeployError( - `Invalid credential spec or corrupted state for credential: ${ - stateCredential?.name || specCredential?.name - } (${stateCredential?.owner || specCredential?.owner})`, - 'VALIDATION_ERROR' - ); - } - ) - ); - - const nextWorkflows = Object.fromEntries( - splitZip(oldState.workflows, spec.workflows).map( - ([workflowKey, stateWorkflow, specWorkflow]) => { - const nextJobs = mergeJobs( - nextCredentials, - stateWorkflow?.jobs || {}, - specWorkflow?.jobs || {} - ); + // Prepare the next state, based on the current state and the spec. + export function mergeSpecIntoState( + oldState: ProjectState, + spec: ProjectSpec, + logger?: Logger + ): ProjectState { + const nextCredentials = Object.fromEntries( + splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( + ([credentialKey, stateCredential, specCredential]) => { + if (specCredential && !stateCredential) { + return [ + credentialKey, + { + id: crypto.randomUUID(), + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + if (specCredential && stateCredential) { + return [ + credentialKey, + { + id: stateCredential.id, + name: specCredential.name, + owner: specCredential.owner, + }, + ]; + } + + throw new DeployError( + `Invalid credential spec or corrupted state for credential: ${ + stateCredential?.name || specCredential?.name + } (${stateCredential?.owner || specCredential?.owner})`, + 'VALIDATION_ERROR' + ); + } + ) + ); + + const nextWorkflows = Object.fromEntries( + splitZip(oldState.workflows, spec.workflows).map( + ([workflowKey, stateWorkflow, specWorkflow]) => { + const nextJobs = mergeJobs( + nextCredentials, + stateWorkflow?.jobs || {}, + specWorkflow?.jobs || {} + ); - const nextTriggers = mergeTriggers( - stateWorkflow?.triggers || {}, - specWorkflow?.triggers || {} - ); + const nextTriggers = mergeTriggers( + stateWorkflow?.triggers || {}, + specWorkflow?.triggers || {} + ); - const nextEdges = mergeEdges( - deepClone({ jobs: nextJobs, triggers: nextTriggers }), - stateWorkflow?.edges || {}, - specWorkflow?.edges || {} - ); + const nextEdges = mergeEdges( + deepClone({ jobs: nextJobs, triggers: nextTriggers }), + stateWorkflow?.edges || {}, + specWorkflow?.edges || {} + ); + + if (specWorkflow && isEmpty(stateWorkflow || {})) { + return [ + workflowKey, + { + id: crypto.randomUUID(), + name: specWorkflow.name, + jobs: nextJobs, + triggers: nextTriggers, + edges: nextEdges, + }, + ]; + } + + if (!specWorkflow && !isEmpty(stateWorkflow || {})) { + logger?.error('Critical error! Cannot continue'); + logger?.error( + 'Workflow found in project state but not spec:', + stateWorkflow?.name + ? `${stateWorkflow.name} (${stateWorkflow?.id})` + : stateWorkflow?.id + ); + process.exit(1); + } - if (specWorkflow && isEmpty(stateWorkflow || {})) { return [ workflowKey, { - id: crypto.randomUUID(), - name: specWorkflow.name, + ...stateWorkflow, + id: stateWorkflow!.id, + name: specWorkflow!.name, jobs: nextJobs, triggers: nextTriggers, edges: nextEdges, }, ]; } + ) + ); + + const projectState: Partial = { + ...oldState, + id: oldState.id || crypto.randomUUID(), + name: spec.name, + workflows: nextWorkflows, + project_credentials: nextCredentials, + }; - if (!specWorkflow && !isEmpty(stateWorkflow || {})) { - logger?.error('Critical error! Cannot continue'); - logger?.error( - 'Workflow found in project state but not spec:', - stateWorkflow?.name - ? `${stateWorkflow.name} (${stateWorkflow?.id})` - : stateWorkflow?.id - ); - process.exit(1); + if (spec.description) projectState.description = spec.description; + + return projectState as ProjectState; + } + + export function getStateFromProjectPayload( + project: ProjectPayload + ): ProjectState { + const workflows = reduceByKey('name', project.workflows, (wf) => { + const { triggers, jobs, edges, ...workflowData } = wf; + const stateWorkflow = { + ...workflowData, + } as Partial; + + stateWorkflow.triggers = reduceByKey('type', wf.triggers!); + stateWorkflow.jobs = reduceByKey('name', wf.jobs); + stateWorkflow.edges = wf.edges.reduce((obj, edge) => { + let sourceName; + if (edge.source_trigger_id) { + const t = wf.triggers.find((t) => t.id === edge.source_trigger_id)!; + sourceName = t.type; + } else { + const job = wf.jobs.find((e) => e.id === edge.source_job_id)!; + sourceName = job.name; } + const target = wf.jobs.find((j) => j.id === edge.target_job_id)!; - return [ - workflowKey, - { - ...stateWorkflow, - id: stateWorkflow!.id, - name: specWorkflow!.name, - jobs: nextJobs, - triggers: nextTriggers, - edges: nextEdges, - }, - ]; - } - ) - ); + const name = hyphenate(`${sourceName}->${hyphenate(target.name)}`); + obj[name] = edge; + return obj; + }, {} as Record); - const projectState: Partial = { - ...oldState, - id: oldState.id || crypto.randomUUID(), - name: spec.name, - workflows: nextWorkflows, - project_credentials: nextCredentials, - }; + return stateWorkflow as WorkflowState; + }); - if (spec.description) projectState.description = spec.description; + const project_credentials = (project.project_credentials || []).reduce( + (acc, credential) => { + const key = hyphenate(`${credential.owner} ${credential.name}`); + acc[key] = credential; + return acc; + }, + {} as Record + ); - return projectState as ProjectState; -} + return { + ...project, + project_credentials, + workflows, + }; + } -export function getStateFromProjectPayload( - project: ProjectPayload -): ProjectState { - const workflows = reduceByKey('name', project.workflows, (wf) => { - const { triggers, jobs, edges, ...workflowData } = wf; - const stateWorkflow = { - ...workflowData, - } as Partial; - - stateWorkflow.triggers = reduceByKey('type', wf.triggers!); - stateWorkflow.jobs = reduceByKey('name', wf.jobs); - stateWorkflow.edges = wf.edges.reduce((obj, edge) => { - let sourceName; - if (edge.source_trigger_id) { - const t = wf.triggers.find((t) => t.id === edge.source_trigger_id)!; - sourceName = t.type; - } else { - const job = wf.jobs.find((e) => e.id === edge.source_job_id)!; - sourceName = job.name; - } - const target = wf.jobs.find((j) => j.id === edge.target_job_id)!; - - const name = hyphenate(`${sourceName}->${hyphenate(target.name)}`); - obj[name] = edge; - return obj; - }, {} as Record); - - return stateWorkflow as WorkflowState; - }); - - const project_credentials = (project.project_credentials || []).reduce( - (acc, credential) => { - const key = hyphenate(`${credential.owner} ${credential.name}`); - acc[key] = credential; - return acc; - }, - {} as Record - ); - - return { - ...project, - project_credentials, - workflows, - }; -} + // Maps the server response to the state, merging the two together. + // The state object is keyed by strings, while the server response is a + // list of objects. + export function mergeProjectPayloadIntoState( + state: ProjectState, + project: ProjectPayload + ): ProjectState { + // TODO: should be raise an error if either the state or the server response + // doesn't match? and/or when the server response is missing an item? + + const nextWorkflows = Object.fromEntries( + idKeyPairs(project.workflows, state.workflows).map( + ([key, nextWorkflow, _state]) => { + const { id, name } = nextWorkflow; + + const jobs = Object.fromEntries( + idKeyPairs(nextWorkflow.jobs, state.workflows[key].jobs).map( + ([key, nextJob, _state]) => [key, nextJob] + ) + ); + const triggers = Object.fromEntries( + idKeyPairs( + nextWorkflow.triggers, + state.workflows[key].triggers + ).map(([key, nextTrigger, _state]) => [key, nextTrigger]) + ); + const edges = Object.fromEntries( + idKeyPairs(nextWorkflow.edges, state.workflows[key].edges).map( + ([key, nextEdge, _state]) => [key, nextEdge] + ) + ); -// Maps the server response to the state, merging the two together. -// The state object is keyed by strings, while the server response is a -// list of objects. -export function mergeProjectPayloadIntoState( - state: ProjectState, - project: ProjectPayload -): ProjectState { - // TODO: should be raise an error if either the state or the server response - // doesn't match? and/or when the server response is missing an item? - - const nextWorkflows = Object.fromEntries( - idKeyPairs(project.workflows, state.workflows).map( - ([key, nextWorkflow, _state]) => { - const { id, name } = nextWorkflow; - - const jobs = Object.fromEntries( - idKeyPairs(nextWorkflow.jobs, state.workflows[key].jobs).map( - ([key, nextJob, _state]) => [key, nextJob] - ) - ); - const triggers = Object.fromEntries( - idKeyPairs(nextWorkflow.triggers, state.workflows[key].triggers).map( - ([key, nextTrigger, _state]) => [key, nextTrigger] - ) - ); - const edges = Object.fromEntries( - idKeyPairs(nextWorkflow.edges, state.workflows[key].edges).map( - ([key, nextEdge, _state]) => [key, nextEdge] - ) - ); + return [ + key, + { + ...nextWorkflow, + id, + name, + jobs, + triggers, + edges, + }, + ]; + } + ) + ); + + const nextCredentials = Object.fromEntries( + idKeyPairs( + project.project_credentials || {}, + state.project_credentials || {} + ).map(([key, nextCredential, _state]) => { + return [key, nextCredential]; + }) + ); - return [ - key, - { - ...nextWorkflow, - id, - name, - jobs, - triggers, - edges, - }, - ]; - } - ) - ); - - const nextCredentials = Object.fromEntries( - idKeyPairs( - project.project_credentials || {}, - state.project_credentials || {} - ).map(([key, nextCredential, _state]) => { - return [key, nextCredential]; - }) - ); - - return { - ...project, - project_credentials: nextCredentials, - workflows: nextWorkflows, - }; -} + return { + ...project, + project_credentials: nextCredentials, + workflows: nextWorkflows, + }; + } -function idKeyPairs

( - projectItems: P[], - stateItems: Record -): [key: string, projectItem: P, stateItem: S][] { - let pairs: [string, P, S][] = []; - for (const projectItem of projectItems) { - for (const [key, stateItem] of Object.entries(stateItems)) { - if (projectItem.id === stateItem.id) { - pairs.push([key, projectItem, stateItem]); + function idKeyPairs

( + projectItems: P[], + stateItems: Record + ): [key: string, projectItem: P, stateItem: S][] { + let pairs: [string, P, S][] = []; + for (const projectItem of projectItems) { + for (const [key, stateItem] of Object.entries(stateItems)) { + if (projectItem.id === stateItem.id) { + pairs.push([key, projectItem, stateItem]); + } } } - } - return pairs; -} + return pairs; + } -export function toProjectPayload(state: ProjectState): ProjectPayload { - // convert the state into a payload that can be sent to the server - // the server expects lists of jobs, triggers, and edges, so we need to - // convert the keyed objects into lists. + export function toProjectPayload(state: ProjectState): ProjectPayload { + // convert the state into a payload that can be sent to the server + // the server expects lists of jobs, triggers, and edges, so we need to + // convert the keyed objects into lists. + + const workflows: ProjectPayload['workflows'] = Object.values( + state.workflows + ).map((workflow) => { + return { + ...workflow, + jobs: Object.values(workflow.jobs), + triggers: Object.values(workflow.triggers), + edges: Object.values(workflow.edges), + }; + }); + + const project_credentials: ProjectPayload['project_credentials'] = + Object.values(state.project_credentials); - const workflows: ProjectPayload['workflows'] = Object.values( - state.workflows - ).map((workflow) => { return { - ...workflow, - jobs: Object.values(workflow.jobs), - triggers: Object.values(workflow.triggers), - edges: Object.values(workflow.edges), + ...state, + project_credentials, + workflows, }; - }); - - const project_credentials: ProjectPayload['project_credentials'] = - Object.values(state.project_credentials); - - return { - ...state, - project_credentials, - workflows, - }; + } } From e89d0be2c5f6bbd6eaeefadbf39b317db1d4e655 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Wed, 14 Aug 2024 13:57:03 +0300 Subject: [PATCH 4/7] rebase issues --- packages/deploy/src/stateTransform.ts | 777 +++++++++++++------------- 1 file changed, 383 insertions(+), 394 deletions(-) diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index 0ba918cab..9f609ae8a 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -29,458 +29,447 @@ function stringifyJobBody(body: SpecJobBody): string { return body; } } - function getStateJobCredential( - specJobCredential: undefined | null | string, - stateCredentials: ProjectState['project_credentials'] - ): undefined | null | string { - if ( - specJobCredential && - typeof stateCredentials[specJobCredential] === 'undefined' - ) { + +function getStateJobCredential( + specJobCredential: string, + stateCredentials: ProjectState['project_credentials'] +): string { + if (!stateCredentials[specJobCredential]) { + throw new DeployError( + `Could not find a credential with name: ${specJobCredential}`, + 'VALIDATION_ERROR' + ); + } + + return stateCredentials[specJobCredential].id; +} + +function mergeJobs( + credentials: ProjectState['project_credentials'], + stateJobs: WorkflowState['jobs'], + specJobs: WorkflowSpec['jobs'] +): WorkflowState['jobs'] { + return Object.fromEntries( + splitZip(stateJobs, specJobs || {}).map(([jobKey, stateJob, specJob]) => { + if (specJob && !stateJob) { + return [ + jobKey, + { + id: crypto.randomUUID(), + name: specJob.name, + adaptor: specJob.adaptor, + body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && + getStateJobCredential(specJob.credential, credentials), + }, + ]; + } + + if (!specJob && stateJob) { + return [jobKey, { id: stateJob.id, delete: true }]; + } + + if (specJob && stateJob) { + return [ + jobKey, + { + id: stateJob.id, + name: specJob.name, + adaptor: specJob.adaptor, + body: stringifyJobBody(specJob.body), + project_credential_id: + specJob.credential && + getStateJobCredential(specJob.credential, credentials), + }, + ]; + } + throw new DeployError( - `Could not find a credential with name: ${specJobCredential}`, + `Invalid job spec or corrupted state for job with key: ${String( + jobKey + )}`, 'VALIDATION_ERROR' ); - } + }) + ); +} - return specJobCredential && stateCredentials[specJobCredential].id; +// Given two objects, find the value of a key in the first object, or the second +// object, falling back to a default value. +function pickValue( + first: Record, + second: Record, + key: string, + defaultValue: any +): any { + if (key in first) { + return first[key]; } - function mergeJobs( - credentials: ProjectState['project_credentials'], - stateJobs: WorkflowState['jobs'], - specJobs: WorkflowSpec['jobs'] - ): WorkflowState['jobs'] { - return Object.fromEntries( - splitZip(stateJobs, specJobs || {}).map(([jobKey, stateJob, specJob]) => { - if (specJob && !stateJob) { + if (key in second) { + return second[key]; + } + + return defaultValue; +} + +function mergeTriggers( + stateTriggers: WorkflowState['triggers'], + specTriggers: WorkflowSpec['triggers'] +): WorkflowState['triggers'] { + return Object.fromEntries( + splitZip(stateTriggers, specTriggers!).map( + ([triggerKey, stateTrigger, specTrigger]) => { + if (specTrigger && !stateTrigger) { return [ - jobKey, + triggerKey, { id: crypto.randomUUID(), - name: specJob.name, - adaptor: specJob.adaptor, - body: stringifyJobBody(specJob.body), - project_credential_id: getStateJobCredential( - specJob.credential, - credentials - ), + ...pickKeys(specTrigger, ['type', 'enabled']), + ...(specTrigger.type === 'cron' + ? { cron_expression: specTrigger.cron_expression } + : {}), }, ]; } - if (!specJob && stateJob) { - return [jobKey, { id: stateJob.id, delete: true }]; + if (!specTrigger && stateTrigger) { + return [triggerKey, { id: stateTrigger!.id, delete: true }]; } - if (specJob && stateJob) { - return [ - jobKey, + // prefer spec, but use state if spec is missing, or default + return [ + triggerKey, + { + id: stateTrigger!.id, + ...{ + type: pickValue(specTrigger!, stateTrigger!, 'type', 'webhook'), + enabled: pickValue(specTrigger!, stateTrigger!, 'enabled', true), + ...(specTrigger!.type === 'cron' + ? { cron_expression: specTrigger!.cron_expression } + : {}), + }, + }, + ]; + } + ) + ); +} + +function mergeEdges( + { jobs, triggers }: Pick, + stateEdges: WorkflowState['edges'], + specEdges: WorkflowSpec['edges'] +): WorkflowState['edges'] { + return Object.fromEntries( + splitZip(stateEdges, specEdges || {}).map( + ([edgeKey, stateEdge, specEdge]) => { + // build a 'new edge', based off the spec and existing jobs and triggers + function convertToStateEdge( + jobs: WorkflowState['jobs'], + triggers: WorkflowState['triggers'], + specEdge: SpecEdge, + id: string + ): StateEdge { + const edge: StateEdge = assignIfTruthy( { - id: stateJob.id, - name: specJob.name, - adaptor: specJob.adaptor, - body: stringifyJobBody(specJob.body), - project_credential_id: getStateJobCredential( - specJob.credential, - credentials - ), + id, + condition_type: specEdge.condition_type ?? null, + target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '', + enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true), }, - ]; - } + { + condition_type: specEdge.condition_type, + condition_expression: specEdge.condition_expression, + condition_label: specEdge.condition_label, + source_job_id: jobs[specEdge.source_job ?? -1]?.id, + source_trigger_id: triggers[specEdge.source_trigger ?? -1]?.id, + } + ); - throw new DeployError( - `Invalid job spec or corrupted state for job with key: ${String( - jobKey - )}`, - 'VALIDATION_ERROR' - ); - }) - ); - } + return edge; + } - // Given two objects, find the value of a key in the first object, or the second - // object, falling back to a default value. - function pickValue( - first: Record, - second: Record, - key: string, - defaultValue: any - ): any { - if (key in first) { - return first[key]; - } + if (specEdge && !stateEdge) { + return [ + edgeKey, + convertToStateEdge(jobs, triggers, specEdge, crypto.randomUUID()), + ]; + } - if (key in second) { - return second[key]; - } + if (!specEdge && stateEdge) { + return [edgeKey, { id: stateEdge.id, delete: true }]; + } - return defaultValue; - } + return [ + edgeKey, + convertToStateEdge(jobs, triggers, specEdge!, stateEdge!.id), + ]; + } + ) + ); +} - function mergeTriggers( - stateTriggers: WorkflowState['triggers'], - specTriggers: WorkflowSpec['triggers'] - ): WorkflowState['triggers'] { - return Object.fromEntries( - splitZip(stateTriggers, specTriggers!).map( - ([triggerKey, stateTrigger, specTrigger]) => { - if (specTrigger && !stateTrigger) { - return [ - triggerKey, - { - id: crypto.randomUUID(), - ...pickKeys(specTrigger, ['type', 'enabled']), - ...(specTrigger.type === 'cron' - ? { cron_expression: specTrigger.cron_expression } - : {}), - }, - ]; - } - - if (!specTrigger && stateTrigger) { - return [triggerKey, { id: stateTrigger!.id, delete: true }]; - } - - // prefer spec, but use state if spec is missing, or default +// Prepare the next state, based on the current state and the spec. +export function mergeSpecIntoState( + oldState: ProjectState, + spec: ProjectSpec, + logger?: Logger +): ProjectState { + const nextCredentials = Object.fromEntries( + splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( + ([credentialKey, stateCredential, specCredential]) => { + if (specCredential && !stateCredential) { return [ - triggerKey, + credentialKey, { - id: stateTrigger!.id, - ...{ - type: pickValue(specTrigger!, stateTrigger!, 'type', 'webhook'), - enabled: pickValue( - specTrigger!, - stateTrigger!, - 'enabled', - true - ), - ...(specTrigger!.type === 'cron' - ? { cron_expression: specTrigger!.cron_expression } - : {}), - }, + id: crypto.randomUUID(), + name: specCredential.name, + owner: specCredential.owner, }, ]; } - ) - ); - } - - function mergeEdges( - { jobs, triggers }: Pick, - stateEdges: WorkflowState['edges'], - specEdges: WorkflowSpec['edges'] - ): WorkflowState['edges'] { - return Object.fromEntries( - splitZip(stateEdges, specEdges || {}).map( - ([edgeKey, stateEdge, specEdge]) => { - // build a 'new edge', based off the spec and existing jobs and triggers - function convertToStateEdge( - jobs: WorkflowState['jobs'], - triggers: WorkflowState['triggers'], - specEdge: SpecEdge, - id: string - ): StateEdge { - const edge: StateEdge = assignIfTruthy( - { - id, - condition_type: specEdge.condition_type ?? null, - target_job_id: jobs[specEdge.target_job ?? -1]?.id ?? '', - enabled: pickValue(specEdge, stateEdge || {}, 'enabled', true), - }, - { - condition_type: specEdge.condition_type, - condition_expression: specEdge.condition_expression, - condition_label: specEdge.condition_label, - source_job_id: jobs[specEdge.source_job ?? -1]?.id, - source_trigger_id: triggers[specEdge.source_trigger ?? -1]?.id, - } - ); - - return edge; - } - - if (specEdge && !stateEdge) { - return [ - edgeKey, - convertToStateEdge(jobs, triggers, specEdge, crypto.randomUUID()), - ]; - } - - if (!specEdge && stateEdge) { - return [edgeKey, { id: stateEdge.id, delete: true }]; - } + if (specCredential && stateCredential) { return [ - edgeKey, - convertToStateEdge(jobs, triggers, specEdge!, stateEdge!.id), + credentialKey, + { + id: stateCredential.id, + name: specCredential.name, + owner: specCredential.owner, + }, ]; } - ) - ); - } - - // Prepare the next state, based on the current state and the spec. - export function mergeSpecIntoState( - oldState: ProjectState, - spec: ProjectSpec, - logger?: Logger - ): ProjectState { - const nextCredentials = Object.fromEntries( - splitZip(oldState.project_credentials || {}, spec.credentials || {}).map( - ([credentialKey, stateCredential, specCredential]) => { - if (specCredential && !stateCredential) { - return [ - credentialKey, - { - id: crypto.randomUUID(), - name: specCredential.name, - owner: specCredential.owner, - }, - ]; - } - - if (specCredential && stateCredential) { - return [ - credentialKey, - { - id: stateCredential.id, - name: specCredential.name, - owner: specCredential.owner, - }, - ]; - } - - throw new DeployError( - `Invalid credential spec or corrupted state for credential: ${ - stateCredential?.name || specCredential?.name - } (${stateCredential?.owner || specCredential?.owner})`, - 'VALIDATION_ERROR' - ); - } - ) - ); - const nextWorkflows = Object.fromEntries( - splitZip(oldState.workflows, spec.workflows).map( - ([workflowKey, stateWorkflow, specWorkflow]) => { - const nextJobs = mergeJobs( - nextCredentials, - stateWorkflow?.jobs || {}, - specWorkflow?.jobs || {} - ); - - const nextTriggers = mergeTriggers( - stateWorkflow?.triggers || {}, - specWorkflow?.triggers || {} - ); + throw new DeployError( + `Invalid credential spec or corrupted state for credential: ${ + stateCredential?.name || specCredential?.name + } (${stateCredential?.owner || specCredential?.owner})`, + 'VALIDATION_ERROR' + ); + } + ) + ); + + const nextWorkflows = Object.fromEntries( + splitZip(oldState.workflows, spec.workflows).map( + ([workflowKey, stateWorkflow, specWorkflow]) => { + const nextJobs = mergeJobs( + nextCredentials, + stateWorkflow?.jobs || {}, + specWorkflow?.jobs || {} + ); - const nextEdges = mergeEdges( - deepClone({ jobs: nextJobs, triggers: nextTriggers }), - stateWorkflow?.edges || {}, - specWorkflow?.edges || {} - ); + const nextTriggers = mergeTriggers( + stateWorkflow?.triggers || {}, + specWorkflow?.triggers || {} + ); - if (specWorkflow && isEmpty(stateWorkflow || {})) { - return [ - workflowKey, - { - id: crypto.randomUUID(), - name: specWorkflow.name, - jobs: nextJobs, - triggers: nextTriggers, - edges: nextEdges, - }, - ]; - } - - if (!specWorkflow && !isEmpty(stateWorkflow || {})) { - logger?.error('Critical error! Cannot continue'); - logger?.error( - 'Workflow found in project state but not spec:', - stateWorkflow?.name - ? `${stateWorkflow.name} (${stateWorkflow?.id})` - : stateWorkflow?.id - ); - process.exit(1); - } + const nextEdges = mergeEdges( + deepClone({ jobs: nextJobs, triggers: nextTriggers }), + stateWorkflow?.edges || {}, + specWorkflow?.edges || {} + ); + if (specWorkflow && isEmpty(stateWorkflow || {})) { return [ workflowKey, { - ...stateWorkflow, - id: stateWorkflow!.id, - name: specWorkflow!.name, + id: crypto.randomUUID(), + name: specWorkflow.name, jobs: nextJobs, triggers: nextTriggers, edges: nextEdges, }, ]; } - ) - ); - - const projectState: Partial = { - ...oldState, - id: oldState.id || crypto.randomUUID(), - name: spec.name, - workflows: nextWorkflows, - project_credentials: nextCredentials, - }; - if (spec.description) projectState.description = spec.description; + if (!specWorkflow && !isEmpty(stateWorkflow || {})) { + logger?.error('Critical error! Cannot continue'); + logger?.error( + 'Workflow found in project state but not spec:', + stateWorkflow?.name + ? `${stateWorkflow.name} (${stateWorkflow?.id})` + : stateWorkflow?.id + ); + process.exit(1); + } - return projectState as ProjectState; - } + return [ + workflowKey, + { + ...stateWorkflow, + id: stateWorkflow!.id, + name: specWorkflow!.name, + jobs: nextJobs, + triggers: nextTriggers, + edges: nextEdges, + }, + ]; + } + ) + ); - export function getStateFromProjectPayload( - project: ProjectPayload - ): ProjectState { - const workflows = reduceByKey('name', project.workflows, (wf) => { - const { triggers, jobs, edges, ...workflowData } = wf; - const stateWorkflow = { - ...workflowData, - } as Partial; - - stateWorkflow.triggers = reduceByKey('type', wf.triggers!); - stateWorkflow.jobs = reduceByKey('name', wf.jobs); - stateWorkflow.edges = wf.edges.reduce((obj, edge) => { - let sourceName; - if (edge.source_trigger_id) { - const t = wf.triggers.find((t) => t.id === edge.source_trigger_id)!; - sourceName = t.type; - } else { - const job = wf.jobs.find((e) => e.id === edge.source_job_id)!; - sourceName = job.name; - } - const target = wf.jobs.find((j) => j.id === edge.target_job_id)!; - - const name = hyphenate(`${sourceName}->${hyphenate(target.name)}`); - obj[name] = edge; - return obj; - }, {} as Record); - - return stateWorkflow as WorkflowState; - }); - - const project_credentials = (project.project_credentials || []).reduce( - (acc, credential) => { - const key = hyphenate(`${credential.owner} ${credential.name}`); - acc[key] = credential; - return acc; - }, - {} as Record - ); + const projectState: Partial = { + ...oldState, + id: oldState.id || crypto.randomUUID(), + name: spec.name, + workflows: nextWorkflows, + project_credentials: nextCredentials, + }; - return { - ...project, - project_credentials, - workflows, - }; - } + if (spec.description) projectState.description = spec.description; - // Maps the server response to the state, merging the two together. - // The state object is keyed by strings, while the server response is a - // list of objects. - export function mergeProjectPayloadIntoState( - state: ProjectState, - project: ProjectPayload - ): ProjectState { - // TODO: should be raise an error if either the state or the server response - // doesn't match? and/or when the server response is missing an item? - - const nextWorkflows = Object.fromEntries( - idKeyPairs(project.workflows, state.workflows).map( - ([key, nextWorkflow, _state]) => { - const { id, name } = nextWorkflow; - - const jobs = Object.fromEntries( - idKeyPairs(nextWorkflow.jobs, state.workflows[key].jobs).map( - ([key, nextJob, _state]) => [key, nextJob] - ) - ); - const triggers = Object.fromEntries( - idKeyPairs( - nextWorkflow.triggers, - state.workflows[key].triggers - ).map(([key, nextTrigger, _state]) => [key, nextTrigger]) - ); - const edges = Object.fromEntries( - idKeyPairs(nextWorkflow.edges, state.workflows[key].edges).map( - ([key, nextEdge, _state]) => [key, nextEdge] - ) - ); + return projectState as ProjectState; +} - return [ - key, - { - ...nextWorkflow, - id, - name, - jobs, - triggers, - edges, - }, - ]; - } - ) - ); +export function getStateFromProjectPayload( + project: ProjectPayload +): ProjectState { + const workflows = reduceByKey('name', project.workflows, (wf) => { + const { triggers, jobs, edges, ...workflowData } = wf; + const stateWorkflow = { + ...workflowData, + } as Partial; + + stateWorkflow.triggers = reduceByKey('type', wf.triggers!); + stateWorkflow.jobs = reduceByKey('name', wf.jobs); + stateWorkflow.edges = wf.edges.reduce((obj, edge) => { + let sourceName; + if (edge.source_trigger_id) { + const t = wf.triggers.find((t) => t.id === edge.source_trigger_id)!; + sourceName = t.type; + } else { + const job = wf.jobs.find((e) => e.id === edge.source_job_id)!; + sourceName = job.name; + } + const target = wf.jobs.find((j) => j.id === edge.target_job_id)!; + + const name = hyphenate(`${sourceName}->${hyphenate(target.name)}`); + obj[name] = edge; + return obj; + }, {} as Record); + + return stateWorkflow as WorkflowState; + }); + + const project_credentials = (project.project_credentials || []).reduce( + (acc, credential) => { + const key = hyphenate(`${credential.owner} ${credential.name}`); + acc[key] = credential; + return acc; + }, + {} as Record + ); + + return { + ...project, + project_credentials, + workflows, + }; +} - const nextCredentials = Object.fromEntries( - idKeyPairs( - project.project_credentials || {}, - state.project_credentials || {} - ).map(([key, nextCredential, _state]) => { - return [key, nextCredential]; - }) - ); +// Maps the server response to the state, merging the two together. +// The state object is keyed by strings, while the server response is a +// list of objects. +export function mergeProjectPayloadIntoState( + state: ProjectState, + project: ProjectPayload +): ProjectState { + // TODO: should be raise an error if either the state or the server response + // doesn't match? and/or when the server response is missing an item? + + const nextWorkflows = Object.fromEntries( + idKeyPairs(project.workflows, state.workflows).map( + ([key, nextWorkflow, _state]) => { + const { id, name } = nextWorkflow; + + const jobs = Object.fromEntries( + idKeyPairs(nextWorkflow.jobs, state.workflows[key].jobs).map( + ([key, nextJob, _state]) => [key, nextJob] + ) + ); + const triggers = Object.fromEntries( + idKeyPairs(nextWorkflow.triggers, state.workflows[key].triggers).map( + ([key, nextTrigger, _state]) => [key, nextTrigger] + ) + ); + const edges = Object.fromEntries( + idKeyPairs(nextWorkflow.edges, state.workflows[key].edges).map( + ([key, nextEdge, _state]) => [key, nextEdge] + ) + ); - return { - ...project, - project_credentials: nextCredentials, - workflows: nextWorkflows, - }; - } + return [ + key, + { + ...nextWorkflow, + id, + name, + jobs, + triggers, + edges, + }, + ]; + } + ) + ); + + const nextCredentials = Object.fromEntries( + idKeyPairs( + project.project_credentials || {}, + state.project_credentials || {} + ).map(([key, nextCredential, _state]) => { + return [key, nextCredential]; + }) + ); + + return { + ...project, + project_credentials: nextCredentials, + workflows: nextWorkflows, + }; +} - function idKeyPairs

( - projectItems: P[], - stateItems: Record - ): [key: string, projectItem: P, stateItem: S][] { - let pairs: [string, P, S][] = []; - for (const projectItem of projectItems) { - for (const [key, stateItem] of Object.entries(stateItems)) { - if (projectItem.id === stateItem.id) { - pairs.push([key, projectItem, stateItem]); - } +function idKeyPairs

( + projectItems: P[], + stateItems: Record +): [key: string, projectItem: P, stateItem: S][] { + let pairs: [string, P, S][] = []; + for (const projectItem of projectItems) { + for (const [key, stateItem] of Object.entries(stateItems)) { + if (projectItem.id === stateItem.id) { + pairs.push([key, projectItem, stateItem]); } } - - return pairs; } - export function toProjectPayload(state: ProjectState): ProjectPayload { - // convert the state into a payload that can be sent to the server - // the server expects lists of jobs, triggers, and edges, so we need to - // convert the keyed objects into lists. - - const workflows: ProjectPayload['workflows'] = Object.values( - state.workflows - ).map((workflow) => { - return { - ...workflow, - jobs: Object.values(workflow.jobs), - triggers: Object.values(workflow.triggers), - edges: Object.values(workflow.edges), - }; - }); - - const project_credentials: ProjectPayload['project_credentials'] = - Object.values(state.project_credentials); + return pairs; +} + +export function toProjectPayload(state: ProjectState): ProjectPayload { + // convert the state into a payload that can be sent to the server + // the server expects lists of jobs, triggers, and edges, so we need to + // convert the keyed objects into lists. + const workflows: ProjectPayload['workflows'] = Object.values( + state.workflows + ).map((workflow) => { return { - ...state, - project_credentials, - workflows, + ...workflow, + jobs: Object.values(workflow.jobs), + triggers: Object.values(workflow.triggers), + edges: Object.values(workflow.edges), }; - } + }); + + const project_credentials: ProjectPayload['project_credentials'] = + Object.values(state.project_credentials); + + return { + ...state, + project_credentials, + workflows, + }; } From 41e3c9f44440cb3103fcf60ffc3363b5702c32da Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Wed, 14 Aug 2024 14:19:18 +0300 Subject: [PATCH 5/7] fix rebase ghosts --- packages/deploy/src/pull.ts | 1 + packages/deploy/src/types.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/deploy/src/pull.ts b/packages/deploy/src/pull.ts index 4df7b7f11..58ee3cb31 100644 --- a/packages/deploy/src/pull.ts +++ b/packages/deploy/src/pull.ts @@ -25,6 +25,7 @@ async function getAllSpecJobs( name: specJob.name, adaptor: specJob.adaptor, body: specJob.body, + credential: specJob.credential, }); } } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 37d6bceae..d721541e3 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -15,6 +15,7 @@ export type SpecJobBody = }; export type SpecJob = { + id?: string; name: string; adaptor: string; body: SpecJobBody; From 0d53f9b76ccc096bf703e21d2bd6fd703a3840d3 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 16 Aug 2024 14:40:23 +0100 Subject: [PATCH 6/7] add changeset --- .changeset/breezy-eyes-brake.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/breezy-eyes-brake.md diff --git a/.changeset/breezy-eyes-brake.md b/.changeset/breezy-eyes-brake.md new file mode 100644 index 000000000..48ebd2015 --- /dev/null +++ b/.changeset/breezy-eyes-brake.md @@ -0,0 +1,5 @@ +--- +'@openfn/deploy': minor +--- + +Add support for basic project-credential management (add, associate with jobs) via the CLI From 0d9df9f677891d1cf491393c0bb4f67255720652 Mon Sep 17 00:00:00 2001 From: Taylor Downs Date: Fri, 16 Aug 2024 14:43:56 +0100 Subject: [PATCH 7/7] new versions --- .changeset/breezy-eyes-brake.md | 5 ----- packages/cli/CHANGELOG.md | 7 +++++++ packages/cli/package.json | 2 +- packages/deploy/CHANGELOG.md | 6 ++++++ packages/deploy/package.json | 2 +- pnpm-lock.yaml | 2 -- 6 files changed, 15 insertions(+), 9 deletions(-) delete mode 100644 .changeset/breezy-eyes-brake.md diff --git a/.changeset/breezy-eyes-brake.md b/.changeset/breezy-eyes-brake.md deleted file mode 100644 index 48ebd2015..000000000 --- a/.changeset/breezy-eyes-brake.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/deploy': minor ---- - -Add support for basic project-credential management (add, associate with jobs) via the CLI diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 96dccc599..93f047634 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # @openfn/cli +## 1.8.1 + +### Patch Changes + +- Updated dependencies [0d53f9b] + - @openfn/deploy@0.7.0 + ## 1.8.0 ### Minor Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index dd8498585..7043d9383 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/cli", - "version": "1.8.0", + "version": "1.8.1", "description": "CLI devtools for the openfn toolchain.", "engines": { "node": ">=18", diff --git a/packages/deploy/CHANGELOG.md b/packages/deploy/CHANGELOG.md index a5b4aaab7..ff172b7ea 100644 --- a/packages/deploy/CHANGELOG.md +++ b/packages/deploy/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/deploy +## 0.7.0 + +### Minor Changes + +- 0d53f9b: Add support for basic project-credential management (add, associate with jobs) via the CLI + ## 0.6.0 ### Minor Changes diff --git a/packages/deploy/package.json b/packages/deploy/package.json index c780b5262..ba962c003 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/deploy", - "version": "0.6.0", + "version": "0.7.0", "description": "Deploy projects to Lightning instances", "type": "module", "exports": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af598fe53..b6f2b5bc3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -468,8 +468,6 @@ importers: specifier: ^5.1.6 version: 5.1.6 - packages/engine-multi/tmp/a/b/c: {} - packages/engine-multi/tmp/repo: {} packages/lexicon: