Working first scenario for deploy

- Point typescript at packages dir when looking for definitions

.devcontainer/devcontainer.json

@@ -1,36 +1,36 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker
 {
-	"name": "Default",
-	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/base:bullseye",
+  "name": "Default",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/base:bullseye",
 
-	"features": {
-		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
-			"version": "latest",
-			"enableNonRootDocker": "true",
-			"moby": "true"
-		},
-		"ghcr.io/devcontainers/features/node:1": {
-			"nodeGypDependencies": true,
-			"version": "18"
-		}
-	},
+  "features": {
+    "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {
+      "version": "latest",
+      "enableNonRootDocker": "true",
+      "moby": "true"
+    },
+    "ghcr.io/devcontainers/features/node:1": {
+      "nodeGypDependencies": true,
+      "version": "18"
+    }
+  },
 
-	// Use this environment variable if you need to bind mount your local source code into a new container.
-	"remoteEnv": {
-		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
-	},
+  // Use this environment variable if you need to bind mount your local source code into a new container.
+  "remoteEnv": {
+    "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
+  },
 
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [],
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
 
-	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "npm install -g pnpm"
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "npm install -g pnpm"
 
-	// Configure tool-specific properties.
-	// "customizations": {},
+  // Configure tool-specific properties.
+  // "customizations": {},
 
-	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
-	// "remoteUser": "root"
+  // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+  // "remoteUser": "root"
 }

.prettierignore

@@ -1,2 +1,4 @@
 node_modules/
 *.md
+
+dist/

ava.config.js

@@ -1,19 +1,17 @@
 module.exports = {
   extensions: {
-    ts: "module"
+    ts: 'module',
   },
 
   environmentVariables: {
-    "TS_NODE_TRANSPILE_ONLY": "true"
+    TS_NODE_TRANSPILE_ONLY: 'true',
   },
 
   nodeArguments: [
-    "--loader=ts-node/esm",
-    "--no-warnings", // Disable experimental module warnings
-    "--experimental-vm-modules"
+    '--loader=ts-node/esm',
+    '--no-warnings', // Disable experimental module warnings
+    '--experimental-vm-modules',
   ],
 
-  files: [
-    "test/**/*test.ts"
-  ]
-}
+  files: ['test/**/*test.ts'],
+};

packages/cli/package.json

@@ -46,6 +46,7 @@
   },
   "dependencies": {
     "@openfn/compiler": "workspace:*",
+    "@openfn/deploy": "workspace:*",
     "@openfn/describe-package": "workspace:*",
     "@openfn/logger": "workspace:*",
     "@openfn/runtime": "workspace:*",

packages/cli/src/cli.ts

@@ -1,20 +1,22 @@
 import yargs, { Arguments } from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
-import { repo as repoCommand, install as installCommand } from './repo/command';
-import executeCommand from './execute/command';
 import compileCommand from './compile/command';
-import testCommand from './test/command';
+import deployCommand from './deploy/command';
 import docgenCommand from './docgen/command';
 import docsCommand from './docs/command';
+import executeCommand from './execute/command';
 import metadataCommand from './metadata/command';
 import { Opts } from './options';
+import { install as installCommand, repo as repoCommand } from './repo/command';
+import testCommand from './test/command';
 
 const y = yargs(hideBin(process.argv));
 
 export const cmd = y
   .command(executeCommand as any)
   .command(compileCommand as any)
+  .command(deployCommand as any)
   .command(installCommand) // allow install to run from the top as well as repo
   .command(repoCommand)
   .command(testCommand)

packages/cli/src/deploy/command.ts

@@ -1,24 +1,40 @@
-import yargs, { Arguments } from 'yargs';
-import { ensure } from '../util/command-builders';
+import yargs, { Argv } from 'yargs';
+import { build, ensure, override } from '../util/command-builders';
 import { Opts } from '../options';
 import * as o from '../options';
 
 export type DeployOptions = Required<
   Pick<
     Opts,
-    'command' | 'log' | 'logJson' | 'statePath' | 'projectPath' | 'configPath'
+    | 'command'
+    | 'log'
+    | 'logJson'
+    | 'statePath'
+    | 'projectPath'
+    | 'configPath'
+    | 'confirm'
   >
 >;
 
-const options = [o.logJson, o.statePath, o.projectPath, o.configPath];
+const options = [
+  o.logJson,
+  override(o.statePath, {
+    default: './.state.json',
+  }),
+  o.projectPath,
+  override(o.configPath, {
+    default: './.config.json',
+  }),
+  o.confirm,
+];
 
 const deployCommand = {
   command: 'deploy',
   desc: "Deploy a project's config to a remote Lightning instance",
-  handler: ensure('deploy', options),
-  builder: (yargs: yargs.Argv) => {
-    return yargs.example('deploy', '');
+  builder: (yargs: yargs.Argv<DeployOptions>) => {
+    return build(options, yargs).example('deploy', '');
   },
-} as yargs.CommandModule<DeployOptions>;
+  handler: ensure('deploy', options),
+};
 
 export default deployCommand;

packages/cli/src/deploy/handler.ts

@@ -1,148 +1,64 @@
-import { writeFile } from 'node:fs/promises';
-import { readFileSync, writeFileSync, mkdirSync, rmSync } from 'node:fs';
-import path from 'node:path';
-
-import { Opts } from '../options';
+import { DeployError, deploy, getConfig, validateConfig } from '@openfn/deploy';
 import type { Logger } from '../util/logger';
+import { DeployOptions } from './command';
 
-import { describePackage, PackageDescription } from '@openfn/describe-package';
-import { getNameAndVersion } from '@openfn/runtime';
-
-export type DeployFn = (specifier: string) => Promise<PackageDescription>;
-
-const RETRY_DURATION = 500;
-const RETRY_COUNT = 20;
-
-const TIMEOUT_MS = 1000 * 60;
-
-const actualDeploy: DeployFn = (specifier: string) =>
-  describePackage(specifier, {});
+export type DeployFn = typeof deploy;
 
-// Ensure the path to a .json file exists
-export const ensurePath = (filePath: string) =>
-  mkdirSync(path.dirname(filePath), { recursive: true });
+const actualDeploy: DeployFn = deploy;
 
-export const generatePlaceholder = (path: string) => {
-  writeFileSync(path, `{ "loading": true, "timestamp": ${Date.now()}}`);
-};
-
-const finish = (logger: Logger, resultPath: string) => {
-  logger.success('Done! Docs can be found at:\n');
-  logger.print(`  ${path.resolve(resultPath)}`);
-};
-
-const generateDocs = async (
-  specifier: string,
-  path: string,
-  docgen: DeployFn,
-  logger: Logger
-) => {
-  const result = await docgen(specifier);
-
-  await writeFile(path, JSON.stringify(result, null, 2));
-  finish(logger, path);
-  return path;
-};
+// Flexible `deployFn` interface for testing.
+async function deployHandler<F extends (...args: any) => any>(
+  options: DeployOptions,
+  logger: Logger,
+  deployFn: F
+): Promise<ReturnType<typeof deployFn>>;
 
-const waitForDocs = async (
-  docs: object,
-  path: string,
+async function deployHandler(
+  options: DeployOptions,
   logger: Logger,
-  retryDuration = RETRY_DURATION
-): Promise<string> => {
+  deployFn = actualDeploy
+) {
   try {
-    if (docs.hasOwnProperty('loading')) {
-      // if this is a placeholder... set an interval and wait for JSON to be written
-      // TODO should we watch with chokidar instead? The polling is actually kinda reassuring
-      logger.info('Docs are being loaded by another process. Waiting.');
-      return new Promise((resolve, reject) => {
-        let count = 0;
-        let i = setInterval(() => {
-          logger.info('Waiting..');
-          if (count > RETRY_COUNT) {
-            clearInterval(i);
-            reject(new Error('Timed out waiting for docs to load'));
-          }
-          const updated = JSON.parse(readFileSync(path, 'utf8'));
-          if (!updated.hasOwnProperty('loading')) {
-            logger.info('Docs found!');
-            clearInterval(i);
-            resolve(path);
-          }
-          count++;
-        }, retryDuration);
-      });
-    } else {
-      logger.info(`Docs already written to cache at ${path}`);
-      finish(logger, path);
-      return path;
-      // If we get here the docs have been written, everything is fine
-      // TODO should we sanity check the name and version? Would make sense
-    }
-  } catch (e) {
-    // If something is wrong with the current JSON, abort for now
-    // To be fair it may not matter as we'll write over it anyway
-    // Maybe we should encourge a openfn docs purge <specifier> or something
-    logger.error('Existing doc JSON corrupt. Aborting');
-    throw e;
-  }
-};
+    logger.debug('Deploying with options', JSON.stringify(options, null, 2));
+    const config = await getConfig(options.configPath);
 
-// This function deliberately blocks woth synchronous I/O
-// while it looks to see whether docs need generating
-const deployHandler = (
-  options: Required<Pick<Opts, 'specifier' | 'repoDir'>>,
-  logger: Logger,
-  docgen: DeployFn = actualDeploy,
-  retryDuration = RETRY_DURATION
-): Promise<string | void> => {
-  const { specifier, repoDir } = options;
+    if (options.confirm === false) {
+      config.requireConfirmation = options.confirm;
+    }
 
-  const { version } = getNameAndVersion(specifier);
-  if (!version) {
-    logger.error('Error: No version number detected');
-    logger.error('eg, @openfn/[email protected]');
-    logger.error('Aborting');
-    process.exit(9); // invalid argument
-  }
+    if (process.env['OPENFN_API_KEY']) {
+      logger.info('Using OPENFN_API_KEY environment variable');
+      config.apiKey = process.env['OPENFN_API_KEY'];
+    }
 
-  logger.success(`Generating docs for ${specifier}`); // TODO not success, but a default level info log.
+    if (process.env['OPENFN_ENDPOINT']) {
+      logger.info('Using OPENFN_ENDPOINT environment variable');
+      config.endpoint = process.env['OPENFN_ENDPOINT'];
+    }
 
-  const path = `${repoDir}/docs/${specifier}.json`;
-  ensurePath(path);
+    logger.debug('Deploying with config', config);
+    logger.info(`Deploying`);
 
-  const handleError = () => {
-    // Remove the placeholder
-    logger.info('Removing placeholder');
-    rmSync(path);
-  };
+    validateConfig(config);
 
-  try {
-    const existing = readFileSync(path, 'utf8');
-    const json = JSON.parse(existing);
-    if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) {
-      // If the placeholder is more than TIMEOUT_MS old, remove it and try again
-      logger.info(`Expired placeholder found. Removing.`);
-      rmSync(path);
-      throw new Error('TIMEOUT');
+    const isOk = await deployFn(config, logger);
+    if (isOk) {
+      process.exitCode = 0;
+      logger.info(`Deployed`);
+      return isOk;
+    } else {
+      process.exitCode = 1;
+      return isOk;
     }
-    // Return or wait for the existing docs
-    // If there's a timeout error, don't remove the placeholder
-    return waitForDocs(json, path, logger, retryDuration);
-  } catch (e) {
-    // Generate docs from scratch
-    if (e.message !== 'TIMEOUT') {
-      logger.info(`Docs JSON not found at ${path}`);
+  } catch (error: any) {
+    if (error instanceof DeployError) {
+      logger.error(error.message);
+      process.exitCode = 10;
+      return false;
     }
-    logger.debug('Generating placeholder');
-    generatePlaceholder(path);
 
-    return generateDocs(specifier, path, docgen, logger).catch((e) => {
-      logger.error('Error generating documentation');
-      logger.error(e);
-      handleError();
-    });
+    throw error;
   }
-};
+}
 
 export default deployHandler;