Merge pull request #743 from OpenFn/support-credentials

Support credentials in deploy package

packages/cli/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/cli
 
+## 1.8.1
+
+### Patch Changes
+
+- Updated dependencies [0d53f9b]
+  - @openfn/deploy@0.7.0
+
 ## 1.8.0
 
 ### Minor Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.8.0",
+  "version": "1.8.1",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",

packages/deploy/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @openfn/deploy
 
+## 0.7.0
+
+### Minor Changes
+
+- 0d53f9b: Add support for basic project-credential management (add, associate with jobs) via the CLI
+
 ## 0.6.0
 
 ### Minor Changes

packages/deploy/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/deploy",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Deploy projects to Lightning instances",
   "type": "module",
   "exports": {

packages/deploy/src/pull.ts

@@ -25,6 +25,7 @@ async function getAllSpecJobs(
               name: specJob.name,
               adaptor: specJob.adaptor,
               body: specJob.body,
+              credential: specJob.credential,
             });
         }
       }

packages/deploy/src/stateTransform.ts

@@ -1,6 +1,7 @@
 import crypto from 'crypto';
 import { deepClone } from 'fast-json-patch';
 import {
+  CredentialState,
   ProjectPayload,
   ProjectSpec,
   ProjectState,
@@ -29,7 +30,22 @@ function stringifyJobBody(body: SpecJobBody): string {
   }
 }
 
+function getStateJobCredential(
+  specJobCredential: string,
+  stateCredentials: ProjectState['project_credentials']
+): string {
+  if (!stateCredentials[specJobCredential]) {
+    throw new DeployError(
+      `Could not find a credential with name: ${specJobCredential}`,
+      'VALIDATION_ERROR'
+    );
+  }
+
+  return stateCredentials[specJobCredential].id;
+}
+
 function mergeJobs(
+  credentials: ProjectState['project_credentials'],
   stateJobs: WorkflowState['jobs'],
   specJobs: WorkflowSpec['jobs']
 ): WorkflowState['jobs'] {
@@ -43,6 +59,9 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: stringifyJobBody(specJob.body),
+            project_credential_id:
+              specJob.credential &&
+              getStateJobCredential(specJob.credential, credentials),
           },
         ];
       }
@@ -59,6 +78,9 @@ function mergeJobs(
             name: specJob.name,
             adaptor: specJob.adaptor,
             body: stringifyJobBody(specJob.body),
+            project_credential_id:
+              specJob.credential &&
+              getStateJobCredential(specJob.credential, credentials),
           },
         ];
       }
@@ -195,10 +217,46 @@ export function mergeSpecIntoState(
   spec: ProjectSpec,
   logger?: Logger
 ): ProjectState {
+  const nextCredentials = Object.fromEntries(
+    splitZip(oldState.project_credentials || {}, spec.credentials || {}).map(
+      ([credentialKey, stateCredential, specCredential]) => {
+        if (specCredential && !stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: crypto.randomUUID(),
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        if (specCredential && stateCredential) {
+          return [
+            credentialKey,
+            {
+              id: stateCredential.id,
+              name: specCredential.name,
+              owner: specCredential.owner,
+            },
+          ];
+        }
+
+        throw new DeployError(
+          `Invalid credential spec or corrupted state for credential: ${
+            stateCredential?.name || specCredential?.name
+          } (${stateCredential?.owner || specCredential?.owner})`,
+          'VALIDATION_ERROR'
+        );
+      }
+    )
+  );
+
   const nextWorkflows = Object.fromEntries(
     splitZip(oldState.workflows, spec.workflows).map(
       ([workflowKey, stateWorkflow, specWorkflow]) => {
         const nextJobs = mergeJobs(
+          nextCredentials,
           stateWorkflow?.jobs || {},
           specWorkflow?.jobs || {}
         );
@@ -258,6 +316,7 @@ export function mergeSpecIntoState(
     id: oldState.id || crypto.randomUUID(),
     name: spec.name,
     workflows: nextWorkflows,
+    project_credentials: nextCredentials,
   };
 
   if (spec.description) projectState.description = spec.description;
@@ -295,8 +354,18 @@ export function getStateFromProjectPayload(
     return stateWorkflow as WorkflowState;
   });
 
+  const project_credentials = (project.project_credentials || []).reduce(
+    (acc, credential) => {
+      const key = hyphenate(`${credential.owner} ${credential.name}`);
+      acc[key] = credential;
+      return acc;
+    },
+    {} as Record<string, CredentialState>
+  );
+
   return {
     ...project,
+    project_credentials,
     workflows,
   };
 }
@@ -347,8 +416,18 @@ export function mergeProjectPayloadIntoState(
     )
   );
 
+  const nextCredentials = Object.fromEntries(
+    idKeyPairs(
+      project.project_credentials || {},
+      state.project_credentials || {}
+    ).map(([key, nextCredential, _state]) => {
+      return [key, nextCredential];
+    })
+  );
+
   return {
     ...project,
+    project_credentials: nextCredentials,
     workflows: nextWorkflows,
   };
 }
@@ -385,8 +464,12 @@ export function toProjectPayload(state: ProjectState): ProjectPayload {
     };
   });
 
+  const project_credentials: ProjectPayload['project_credentials'] =
+    Object.values(state.project_credentials);
+
   return {
     ...state,
+    project_credentials,
     workflows,
   };
 }

packages/deploy/src/types.ts

@@ -1,8 +1,9 @@
 export type StateJob = {
-  id?: string;
+  id: string;
   name: string;
   adaptor: string;
   body: string;
+  project_credential_id: string | null;
   delete?: boolean;
 };
 
@@ -18,6 +19,7 @@ export type SpecJob = {
   name: string;
   adaptor: string;
   body: SpecJobBody;
+  credential: string | null;
 };
 
 export type Trigger = {
@@ -57,10 +59,22 @@ export type WorkflowSpec = {
   edges?: Record<string | symbol, SpecEdge>;
 };
 
+export type CredentialSpec = {
+  name: string;
+  owner: string;
+};
+
+export type CredentialState = {
+  id: string;
+  name: string;
+  owner: string;
+};
+
 export interface ProjectSpec {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowSpec>;
+  credentials: Record<string | symbol, CredentialSpec>;
 }
 
 export interface WorkflowState {
@@ -82,12 +96,14 @@ export interface ProjectState {
   name: string;
   description: string;
   workflows: Record<string | symbol, WorkflowState>;
+  project_credentials: Record<string | symbol, CredentialState>;
 }
 
 export interface ProjectPayload {
   id: string;
   name: string;
   description: string;
+  project_credentials: Concrete<CredentialState>[];
   workflows: {
     id: string;
     name: string;

packages/deploy/src/validator.ts

@@ -117,6 +117,12 @@ export async function parseAndValidate(
         }
       }
 
+      if (pair.key && pair.key.value === 'credentials') {
+        if (pair.value.value === null) {
+          return doc.createPair('credentials', {});
+        }
+      }
+
       if (pair.key && pair.key.value === 'jobs') {
         if (pair.value.value === null) {
           errors.push({