Support credentials in deploy package

packages/deploy/src/pull.ts

@@ -25,6 +25,7 @@ async function getAllSpecJobs(
               name: specJob.name,
               adaptor: specJob.adaptor,
               body: specJob.body,
+              credential: specJob.credential,
             });
         }
       }

packages/deploy/src/stateTransform.ts

@@ -1,6 +1,7 @@
 import crypto from 'crypto';
 import { deepClone } from 'fast-json-patch';
 import {
+  CredentialState,
   ProjectPayload,
   ProjectSpec,
   ProjectState,
@@ -29,7 +30,22 @@ function stringifyJobBody(body: SpecJobBody): string {
   }
 }
 
+function getStateJobCredential(
+  specJobCredential: string,
+  stateCredentials: ProjectState['project_credentials']
+): string {
+  if (!stateCredentials[specJobCredential]) {
+    throw new DeployError(
+      `Could not find a credential with name: ${specJobCredential}`,
+      'VALIDATION_ERROR'
+    );
+  }
+
+  return stateCredentials[specJobCredential].id;
+}
+
 function mergeJobs(
+  credentials: ProjectState['project_credentials'],
   stateJobs: WorkflowState['jobs'],
   specJobs: WorkflowSpec['jobs']
 ): WorkflowState['jobs'] {
@@ -43,6 +59,9 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: stringifyJobBody(specJob.body),
+            project_credential_id:
+              specJob.credential &&
+              getStateJobCredential(specJob.credential, credentials),
           },
         ];
       }
@@ -59,6 +78,9 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: stringifyJobBody(specJob.body),
+            project_credential_id:
+              specJob.credential &&
+              getStateJobCredential(specJob.credential, credentials),
           },
         ];
       }
@@ -195,10 +217,46 @@ export function mergeSpecIntoState(
   spec: ProjectSpec,
   logger?: Logger
 ): ProjectState {
+  const nextCredentials = Object.fromEntries(
+    splitZip(oldState.project_credentials || {}, spec.credentials || {}).map(
+      ([credentialKey, stateCredential, specCredential]) => {
+        if (specCredential && !stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: crypto.randomUUID(),
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        if (specCredential && stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: stateCredential.id,
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        throw new DeployError(
+          `Invalid credential spec or corrupted state for credential: ${
+            stateCredential?.name || specCredential?.name
+          } (${stateCredential?.owner || specCredential?.owner})`,
+          'VALIDATION_ERROR'
+        );
+      }
+    )
+  );
+
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
       ([workflowKey, stateWorkflow, specWorkflow]) => {
         const nextJobs = mergeJobs(
+          nextCredentials,
           stateWorkflow?.jobs || {},
           specWorkflow?.jobs || {}
         );
@@ -258,6 +316,7 @@ export function mergeSpecIntoState(
     id: oldState.id || crypto.randomUUID(),
     name: spec.name,
     workflows: nextWorkflows,
+    project_credentials: nextCredentials,
   };
 
   if (spec.description) projectState.description = spec.description;
@@ -295,8 +354,18 @@ export function getStateFromProjectPayload(
     return stateWorkflow as WorkflowState;
   });
 
+  const project_credentials = (project.project_credentials || []).reduce(
+    (acc, credential) => {
+      const key = hyphenate(`${credential.owner} ${credential.name}`);
+      acc[key] = credential;
+      return acc;
+    },
+    {} as Record<string, CredentialState>
+  );
+
   return {
     ...project,
+    project_credentials,
     workflows,
   };
 }
@@ -347,8 +416,18 @@ export function mergeProjectPayloadIntoState(
     )
   );
 
+  const nextCredentials = Object.fromEntries(
+    idKeyPairs(
+      project.project_credentials || {},
+      state.project_credentials || {}
+    ).map(([key, nextCredential, _state]) => {
+      return [key, nextCredential];
+    })
+  );
+
   return {
     ...project,
+    project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
 }
@@ -385,8 +464,12 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
     };
   });
 
+  const project_credentials: ProjectPayload['project_credentials'] =
+    Object.values(state.project_credentials);
+
   return {
     ...state,
+    project_credentials,
     workflows,
   };
 }

packages/deploy/src/types.ts

@@ -1,8 +1,9 @@
 export type StateJob = {
-  id?: string;
+  id: string;
   name: string;
   adaptor: string;
   body: string;
+  project_credential_id: string | null;
   delete?: boolean;
 };
 
@@ -18,6 +19,7 @@ export type SpecJob = {
   name: string;
   adaptor: string;
   body: SpecJobBody;
+  credential: string | null;
 };
 
 export type Trigger = {
@@ -57,10 +59,22 @@ export type WorkflowSpec = {
   edges?: Record<string | symbol, SpecEdge>;
 };
 
+export type CredentialSpec = {
+  name: string;
+  owner: string;
+};
+
+export type CredentialState = {
+  id: string;
+  name: string;
+  owner: string;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
+  credentials: Record<string | symbol, CredentialSpec>;
 }
 
 export interface WorkflowState {
@@ -82,12 +96,14 @@ export interface ProjectState {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowState>;
+  project_credentials: Record<string | symbol, CredentialState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
+  project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;
     name: string;

packages/deploy/src/validator.ts

@@ -117,6 +117,12 @@ export async function parseAndValidate(
         }
       }
 
+      if (pair.key && pair.key.value === 'credentials') {
+        if (pair.value.value === null) {
+          return doc.createPair('credentials', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({

packages/deploy/test/fixtures.ts

@@ -4,6 +4,7 @@ export function fullExampleSpec() {
   return {
     name: 'my project',
     description: 'some helpful description',
+    credentials: {},
     workflows: {
       'workflow-one': {
         name: 'workflow one',
@@ -15,11 +16,13 @@ export function fullExampleSpec() {
               path: 'somefile.js',
               content: '',
             },
+            credential: null,
           },
           'job-b': {
             name: 'job b',
             adaptor: '@openfn/language-common@latest',
             body: '',
+            credential: null,
           },
         },
         triggers: {
@@ -53,6 +56,7 @@ export function fullExampleState() {
     id: 'be156ab1-8426-4151-9a18-4045142f9ec0',
     name: 'my project',
     description: 'some helpful description',
+    project_credentials: {},
     workflows: {
       'workflow-one': {
         id: '8124e88c-566f-472f-be38-363e588af55a',
@@ -63,12 +67,14 @@ export function fullExampleState() {
             name: 'job a',
             adaptor: '@openfn/language-common@latest',
             body: '',
+            project_credential_id: null,
           },
           'job-b': {
             id: 'e1bf76a8-4deb-44ff-9881-fbf676537b37',
             name: 'job b',
             adaptor: '@openfn/language-common@latest',
             body: '',
+            project_credential_id: null,
           },
         },
         triggers: {
@@ -110,6 +116,13 @@ export const lightningProjectPayload = {
   updated_at: '2023-08-25T08:57:31',
   scheduled_deletion: null,
   requires_mfa: false,
+  project_credentials: [
+    {
+      id: '25f48989-d349-4eb8-99c3-923ebba5b116',
+      name: 'Basic Auth',
+      owner: '[email protected]',
+    },
+  ],
   workflows: [
     {
       id: '05fab294-98dc-4d7d-85f3-024b2b0e6897',
@@ -154,24 +167,28 @@ export const lightningProjectPayload = {
           name: 'FHIR standard Data with change',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116',
         },
         {
           id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           name: 'Send to OpenHIM to route to SHR',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
         {
           id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b',
           name: 'Notify CHW upload successful',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
         {
           id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac',
           name: 'Notify CHW upload failed',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
       ],
       triggers: [
@@ -223,6 +240,13 @@ export const lightningProjectState = {
   updated_at: '2023-08-25T08:57:31',
   scheduled_deletion: null,
   requires_mfa: false,
+  project_credentials: {
+    '[email protected]': {
+      id: '25f48989-d349-4eb8-99c3-923ebba5b116',
+      name: 'Basic Auth',
+      owner: '[email protected]',
+    },
+  },
   workflows: {
     'OpenHIE-Workflow': {
       id: '05fab294-98dc-4d7d-85f3-024b2b0e6897',
@@ -267,24 +291,28 @@ export const lightningProjectState = {
           name: 'FHIR standard Data with change',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: '25f48989-d349-4eb8-99c3-923ebba5b116',
         },
         'Send-to-OpenHIM-to-route-to-SHR': {
           id: 'ed3f110a-c800-479b-9576-47bb87e9ad57',
           name: 'Send to OpenHIM to route to SHR',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
         'Notify-CHW-upload-successful': {
           id: 'f76a4faa-b648-4f44-b865-21154fa7ef7b',
           name: 'Notify CHW upload successful',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
         'Notify-CHW-upload-failed': {
           id: 'd7ac4cfa-b900-4e14-80a3-94149589bbac',
           name: 'Notify CHW upload failed',
           body: 'fn(state => state);\n',
           adaptor: '@openfn/language-http@latest',
+          project_credential_id: null,
         },
       },
       triggers: {