support credentials

packages/deploy/src/index.ts

@@ -74,7 +74,7 @@ export async function getState(path: string) {
     return await readState(path);
   } catch (error: any) {
     if (error.code === 'ENOENT') {
-      return { workflows: {} } as ProjectState;
+      return { workflows: {}, project_credentials: {} } as ProjectState;
     } else {
       throw error;
     }

packages/deploy/src/stateTransform.ts

@@ -1,6 +1,9 @@
+// @ts-nocheck
+// @ts-ignore
 import crypto from 'crypto';
 import { deepClone } from 'fast-json-patch';
 import {
+  CredentialState,
   ProjectPayload,
   ProjectSpec,
   ProjectState,
@@ -21,6 +24,7 @@ import { DeployError } from './deployError';
 import { Logger } from '@openfn/logger/dist';
 
 function mergeJobs(
+  credentials: ProjectState['project_credentials'],
   stateJobs: WorkflowState['jobs'],
   specJobs: WorkflowSpec['jobs']
 ): WorkflowState['jobs'] {
@@ -34,6 +38,7 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: specJob.body,
+            project_credential_id: specJob.credential && credentials[specJob.credential]?.id
           },
         ];
       }
@@ -50,6 +55,7 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: specJob.body,
+            project_credential_id: specJob.credential && credentials[specJob.credential]?.id
           },
         ];
       }
@@ -186,10 +192,48 @@ export function mergeSpecIntoState(
   spec: ProjectSpec,
   logger?: Logger
 ): ProjectState {
+  const nextCredentials = Object.fromEntries(
+    splitZip(oldState.project_credentials || {}, spec.credentials || {}).map(
+      ([credentialKey, stateCredential, specCredential]) => {
+        if (specCredential && !stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: crypto.randomUUID(),
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        if (specCredential && stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: stateCredential.id,
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        if (!specCredential && !isEmpty(stateCredential || {})) {
+          logger?.error('Critical error! Cannot continue');
+          logger?.error(
+            'Crdential found in project state but not spec:',
+            `${stateCredential.name} (${stateCredential.owner})`
+          );
+          process.exit(1);
+        }
+      }
+    )
+  );
+
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
       ([workflowKey, stateWorkflow, specWorkflow]) => {
         const nextJobs = mergeJobs(
+          nextCredentials,
           stateWorkflow?.jobs || {},
           specWorkflow?.jobs || {}
         );
@@ -249,13 +293,16 @@ export function mergeSpecIntoState(
     id: oldState.id || crypto.randomUUID(),
     name: spec.name,
     workflows: nextWorkflows,
+    project_credentials: nextCredentials
   };
 
   if (spec.description) projectState.description = spec.description;
 
   return projectState as ProjectState;
 }
 
+// @ts-nocheck
+// @ts-ignore
 export function getStateFromProjectPayload(
   project: ProjectPayload
 ): ProjectState {
@@ -286,8 +333,15 @@ export function getStateFromProjectPayload(
     return stateWorkflow as WorkflowState;
   });
 
+  const project_credentials = project.project_credentials.reduce((acc, credential) => {
+    const key = hyphenate(`${credential.owner} ${credential.name}`);
+    acc[key] = credential;
+    return acc;
+  }, {} as Record<string, CredentialState>);
+
   return {
     ...project,
+    project_credentials,
     workflows,
   };
 }
@@ -338,8 +392,18 @@ export function mergeProjectPayloadIntoState(
     )
   );
 
+  const nextCredentials = Object.fromEntries(
+    idKeyPairs(project.project_credentials, state.project_credentials).map(
+      ([key, nextCredential, _state]) => {
+        return [
+          key,
+          nextCredential];
+      }
+    ));
+
   return {
     ...project,
+    project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
 }
@@ -376,8 +440,13 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
     };
   });
 
+  const project_credentials: ProjectPayload['project_credentials'] = Object.values(
+    state.project_credentials
+  )
+
   return {
     ...state,
+    project_credentials,
     workflows,
   };
 }

packages/deploy/src/types.ts

@@ -1,11 +1,19 @@
-export type Job = {
-  id?: string;
+export type StateJob = {
+  id: string;
   name: string;
   adaptor: string;
   body: string;
+  project_credential_id: string | null,
   delete?: boolean;
 };
 
+export type SpecJob = {
+  name: string;
+  adaptor: string;
+  body: string;
+  credential: string | null;
+}
+
 export type Trigger = {
   id?: string;
   type?: string;
@@ -38,21 +46,33 @@ export type SpecEdge = {
 export type WorkflowSpec = {
   id?: string;
   name: string;
-  jobs?: Record<string | symbol, Job>;
+  jobs?: Record<string | symbol, SpecJob>;
   triggers?: Record<string | symbol, Trigger>;
   edges?: Record<string | symbol, SpecEdge>;
 };
 
+export type CredentialSpec = {
+  name: string;
+  owner: string;
+};
+
+export type CredentialState = {
+  id: string;
+  name: string;
+  owner: string;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
+  credentials: Record<string | symbol, CredentialSpec>;
 }
 
 export interface WorkflowState {
   id: string;
   name: string;
-  jobs: Record<string | symbol, Concrete<Job>>;
+  jobs: Record<string | symbol, Concrete<StateJob>>;
   triggers: Record<string | symbol, Concrete<Trigger>>;
   edges: Record<string | symbol, Concrete<StateEdge>>;
   delete?: boolean;
@@ -68,17 +88,19 @@ export interface ProjectState {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowState>;
+  project_credentials: Record<string | symbol, CredentialState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
+  project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;
     name: string;
     project_id?: string;
-    jobs: Concrete<Job>[];
+    jobs: Concrete<StateJob>[];
     triggers: Concrete<Trigger>[];
     edges: Concrete<StateEdge>[];
   }[];

packages/deploy/src/validator.ts

@@ -111,6 +111,12 @@ export function parseAndValidate(input: string): {
         }
       }
 
+      if (pair.key && pair.key.value === 'credentials') {
+        if (pair.value.value === null) {
+          return doc.createPair('credentials', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({