From c86f85c71a740c9ecb731fb63fd51eccf8c54b9d Mon Sep 17 00:00:00 2001 From: Stuart Corbishley Date: Wed, 7 Jun 2023 08:36:33 +0200 Subject: [PATCH] Working first scenario for deploy - Point typescript at packages dir when looking for definitions --- .devcontainer/devcontainer.json | 52 ++-- .prettierignore | 2 + ava.config.js | 16 +- packages/cli/package.json | 1 + packages/cli/src/cli.ts | 8 +- packages/cli/src/deploy/command.ts | 32 +- packages/cli/src/deploy/handler.ts | 174 +++-------- packages/cli/src/options.ts | 19 +- packages/cli/src/util/command-builders.ts | 8 +- packages/cli/test/deploy/deploy.test.ts | 296 ++++--------------- packages/cli/test/deploy/options.test.ts | 8 +- packages/cli/tsconfig.json | 5 +- packages/deploy/src/client.ts | 124 +++++--- packages/deploy/src/deployError.ts | 16 + packages/deploy/src/index.ts | 161 ++++++---- packages/deploy/src/stateTransform.ts | 57 ++-- packages/deploy/src/types.ts | 3 +- packages/deploy/src/utils.ts | 8 +- packages/deploy/src/validator.ts | 42 ++- packages/deploy/test/changeFormatter.test.ts | 65 ---- packages/deploy/test/fixtures.ts | 2 +- packages/deploy/test/stateTransform.test.ts | 11 +- packages/deploy/test/validator.test.ts | 6 +- packages/deploy/tsconfig.json | 7 +- pnpm-lock.yaml | 5 +- 25 files changed, 504 insertions(+), 624 deletions(-) create mode 100644 packages/deploy/src/deployError.ts delete mode 100644 packages/deploy/test/changeFormatter.test.ts diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 601b03d84..e35a6be6c 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,36 +1,36 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the // README at: https://github.com/devcontainers/templates/tree/main/src/docker-outside-of-docker { - "name": "Default", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/base:bullseye", + "name": "Default", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/base:bullseye", - "features": { - "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { - "version": "latest", - "enableNonRootDocker": "true", - "moby": "true" - }, - "ghcr.io/devcontainers/features/node:1": { - "nodeGypDependencies": true, - "version": "18" - } - }, + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { + "version": "latest", + "enableNonRootDocker": "true", + "moby": "true" + }, + "ghcr.io/devcontainers/features/node:1": { + "nodeGypDependencies": true, + "version": "18" + } + }, - // Use this environment variable if you need to bind mount your local source code into a new container. - "remoteEnv": { - "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" - }, + // Use this environment variable if you need to bind mount your local source code into a new container. + "remoteEnv": { + "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" + }, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "npm install -g pnpm" + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "npm install -g pnpm" - // Configure tool-specific properties. - // "customizations": {}, + // Configure tool-specific properties. + // "customizations": {}, - // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. - // "remoteUser": "root" + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" } diff --git a/.prettierignore b/.prettierignore index d77ae12d0..66b33a8b8 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,2 +1,4 @@ node_modules/ *.md + +dist/ diff --git a/ava.config.js b/ava.config.js index 6460dba66..aff2feea3 100644 --- a/ava.config.js +++ b/ava.config.js @@ -1,19 +1,17 @@ module.exports = { extensions: { - ts: "module" + ts: 'module', }, environmentVariables: { - "TS_NODE_TRANSPILE_ONLY": "true" + TS_NODE_TRANSPILE_ONLY: 'true', }, nodeArguments: [ - "--loader=ts-node/esm", - "--no-warnings", // Disable experimental module warnings - "--experimental-vm-modules" + '--loader=ts-node/esm', + '--no-warnings', // Disable experimental module warnings + '--experimental-vm-modules', ], - files: [ - "test/**/*test.ts" - ] -} \ No newline at end of file + files: ['test/**/*test.ts'], +}; diff --git a/packages/cli/package.json b/packages/cli/package.json index d47b5b269..fc7aa2a99 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -46,6 +46,7 @@ }, "dependencies": { "@openfn/compiler": "workspace:*", + "@openfn/deploy": "workspace:*", "@openfn/describe-package": "workspace:*", "@openfn/logger": "workspace:*", "@openfn/runtime": "workspace:*", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 832f1fd9f..72c3f3d1c 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1,20 +1,22 @@ import yargs, { Arguments } from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { repo as repoCommand, install as installCommand } from './repo/command'; -import executeCommand from './execute/command'; import compileCommand from './compile/command'; -import testCommand from './test/command'; +import deployCommand from './deploy/command'; import docgenCommand from './docgen/command'; import docsCommand from './docs/command'; +import executeCommand from './execute/command'; import metadataCommand from './metadata/command'; import { Opts } from './options'; +import { install as installCommand, repo as repoCommand } from './repo/command'; +import testCommand from './test/command'; const y = yargs(hideBin(process.argv)); export const cmd = y .command(executeCommand as any) .command(compileCommand as any) + .command(deployCommand as any) .command(installCommand) // allow install to run from the top as well as repo .command(repoCommand) .command(testCommand) diff --git a/packages/cli/src/deploy/command.ts b/packages/cli/src/deploy/command.ts index 50123ecbf..87d85e48c 100644 --- a/packages/cli/src/deploy/command.ts +++ b/packages/cli/src/deploy/command.ts @@ -1,24 +1,40 @@ -import yargs, { Arguments } from 'yargs'; -import { ensure } from '../util/command-builders'; +import yargs, { Argv } from 'yargs'; +import { build, ensure, override } from '../util/command-builders'; import { Opts } from '../options'; import * as o from '../options'; export type DeployOptions = Required< Pick< Opts, - 'command' | 'log' | 'logJson' | 'statePath' | 'projectPath' | 'configPath' + | 'command' + | 'log' + | 'logJson' + | 'statePath' + | 'projectPath' + | 'configPath' + | 'confirm' > >; -const options = [o.logJson, o.statePath, o.projectPath, o.configPath]; +const options = [ + o.logJson, + override(o.statePath, { + default: './.state.json', + }), + o.projectPath, + override(o.configPath, { + default: './.config.json', + }), + o.confirm, +]; const deployCommand = { command: 'deploy', desc: "Deploy a project's config to a remote Lightning instance", - handler: ensure('deploy', options), - builder: (yargs: yargs.Argv) => { - return yargs.example('deploy', ''); + builder: (yargs: yargs.Argv) => { + return build(options, yargs).example('deploy', ''); }, -} as yargs.CommandModule; + handler: ensure('deploy', options), +}; export default deployCommand; diff --git a/packages/cli/src/deploy/handler.ts b/packages/cli/src/deploy/handler.ts index 868768dcd..e8221cf86 100644 --- a/packages/cli/src/deploy/handler.ts +++ b/packages/cli/src/deploy/handler.ts @@ -1,148 +1,64 @@ -import { writeFile } from 'node:fs/promises'; -import { readFileSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; -import path from 'node:path'; - -import { Opts } from '../options'; +import { DeployError, deploy, getConfig, validateConfig } from '@openfn/deploy'; import type { Logger } from '../util/logger'; +import { DeployOptions } from './command'; -import { describePackage, PackageDescription } from '@openfn/describe-package'; -import { getNameAndVersion } from '@openfn/runtime'; - -export type DeployFn = (specifier: string) => Promise; - -const RETRY_DURATION = 500; -const RETRY_COUNT = 20; - -const TIMEOUT_MS = 1000 * 60; - -const actualDeploy: DeployFn = (specifier: string) => - describePackage(specifier, {}); +export type DeployFn = typeof deploy; -// Ensure the path to a .json file exists -export const ensurePath = (filePath: string) => - mkdirSync(path.dirname(filePath), { recursive: true }); +const actualDeploy: DeployFn = deploy; -export const generatePlaceholder = (path: string) => { - writeFileSync(path, `{ "loading": true, "timestamp": ${Date.now()}}`); -}; - -const finish = (logger: Logger, resultPath: string) => { - logger.success('Done! Docs can be found at:\n'); - logger.print(` ${path.resolve(resultPath)}`); -}; - -const generateDocs = async ( - specifier: string, - path: string, - docgen: DeployFn, - logger: Logger -) => { - const result = await docgen(specifier); - - await writeFile(path, JSON.stringify(result, null, 2)); - finish(logger, path); - return path; -}; +// Flexible `deployFn` interface for testing. +async function deployHandler any>( + options: DeployOptions, + logger: Logger, + deployFn: F +): Promise>; -const waitForDocs = async ( - docs: object, - path: string, +async function deployHandler( + options: DeployOptions, logger: Logger, - retryDuration = RETRY_DURATION -): Promise => { + deployFn = actualDeploy +) { try { - if (docs.hasOwnProperty('loading')) { - // if this is a placeholder... set an interval and wait for JSON to be written - // TODO should we watch with chokidar instead? The polling is actually kinda reassuring - logger.info('Docs are being loaded by another process. Waiting.'); - return new Promise((resolve, reject) => { - let count = 0; - let i = setInterval(() => { - logger.info('Waiting..'); - if (count > RETRY_COUNT) { - clearInterval(i); - reject(new Error('Timed out waiting for docs to load')); - } - const updated = JSON.parse(readFileSync(path, 'utf8')); - if (!updated.hasOwnProperty('loading')) { - logger.info('Docs found!'); - clearInterval(i); - resolve(path); - } - count++; - }, retryDuration); - }); - } else { - logger.info(`Docs already written to cache at ${path}`); - finish(logger, path); - return path; - // If we get here the docs have been written, everything is fine - // TODO should we sanity check the name and version? Would make sense - } - } catch (e) { - // If something is wrong with the current JSON, abort for now - // To be fair it may not matter as we'll write over it anyway - // Maybe we should encourge a openfn docs purge or something - logger.error('Existing doc JSON corrupt. Aborting'); - throw e; - } -}; + logger.debug('Deploying with options', JSON.stringify(options, null, 2)); + const config = await getConfig(options.configPath); -// This function deliberately blocks woth synchronous I/O -// while it looks to see whether docs need generating -const deployHandler = ( - options: Required>, - logger: Logger, - docgen: DeployFn = actualDeploy, - retryDuration = RETRY_DURATION -): Promise => { - const { specifier, repoDir } = options; + if (options.confirm === false) { + config.requireConfirmation = options.confirm; + } - const { version } = getNameAndVersion(specifier); - if (!version) { - logger.error('Error: No version number detected'); - logger.error('eg, @openfn/language-common@1.7.5'); - logger.error('Aborting'); - process.exit(9); // invalid argument - } + if (process.env['OPENFN_API_KEY']) { + logger.info('Using OPENFN_API_KEY environment variable'); + config.apiKey = process.env['OPENFN_API_KEY']; + } - logger.success(`Generating docs for ${specifier}`); // TODO not success, but a default level info log. + if (process.env['OPENFN_ENDPOINT']) { + logger.info('Using OPENFN_ENDPOINT environment variable'); + config.endpoint = process.env['OPENFN_ENDPOINT']; + } - const path = `${repoDir}/docs/${specifier}.json`; - ensurePath(path); + logger.debug('Deploying with config', config); + logger.info(`Deploying`); - const handleError = () => { - // Remove the placeholder - logger.info('Removing placeholder'); - rmSync(path); - }; + validateConfig(config); - try { - const existing = readFileSync(path, 'utf8'); - const json = JSON.parse(existing); - if (json && json.timeout && Date.now() - json.timeout >= TIMEOUT_MS) { - // If the placeholder is more than TIMEOUT_MS old, remove it and try again - logger.info(`Expired placeholder found. Removing.`); - rmSync(path); - throw new Error('TIMEOUT'); + const isOk = await deployFn(config, logger); + if (isOk) { + process.exitCode = 0; + logger.info(`Deployed`); + return isOk; + } else { + process.exitCode = 1; + return isOk; } - // Return or wait for the existing docs - // If there's a timeout error, don't remove the placeholder - return waitForDocs(json, path, logger, retryDuration); - } catch (e) { - // Generate docs from scratch - if (e.message !== 'TIMEOUT') { - logger.info(`Docs JSON not found at ${path}`); + } catch (error: any) { + if (error instanceof DeployError) { + logger.error(error.message); + process.exitCode = 10; + return false; } - logger.debug('Generating placeholder'); - generatePlaceholder(path); - return generateDocs(specifier, path, docgen, logger).catch((e) => { - logger.error('Error generating documentation'); - logger.error(e); - handleError(); - }); + throw error; } -}; +} export default deployHandler; diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index 38f18cbb7..c0f32b3ee 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -17,6 +17,7 @@ export type Opts = { adaptors?: string[]; autoinstall?: boolean; compile?: boolean; + confirm?: boolean; configPath?: string; expandAdaptors?: boolean; // for unit tests really force?: boolean; @@ -115,10 +116,21 @@ export const compile: CLIOption = { }, }; +export const confirm: CLIOption = { + name: 'no-confirm', + yargs: { + boolean: true, + description: "Skip confirmation prompts (e.g. 'Are you sure?')", + }, + ensure: (opts) => { + setDefaultValue(opts, 'confirm', true); + }, +}; + export const configPath: CLIOption = { name: 'config', yargs: { - alias: ['c'], + alias: ['c', 'config-path'], description: 'The location of your config file', default: './config.json', }, @@ -323,13 +335,8 @@ export const statePath: CLIOption = { description: 'Path to the state file', }, ensure: (opts) => { - console.log(opts); - // remove the alias delete (opts as { s?: string }).s; - // if (opts.command == 'deploy') { - // setDefaultValue(opts, 'statePath', './state.json'); - // } }, }; diff --git a/packages/cli/src/util/command-builders.ts b/packages/cli/src/util/command-builders.ts index 6388ca85a..b6d42246c 100644 --- a/packages/cli/src/util/command-builders.ts +++ b/packages/cli/src/util/command-builders.ts @@ -10,8 +10,12 @@ const expandYargs = (y: {} | (() => any)) => { }; // build helper to chain options -export const build = (opts: CLIOption[], yargs: yargs.Argv) => - opts.reduce((_y, o) => yargs.option(o.name, expandYargs(o.yargs)), yargs); +export function build(opts: CLIOption[], yargs: yargs.Argv) { + return opts.reduce( + (_y, o) => yargs.option(o.name, expandYargs(o.yargs)), + yargs + ); +} // Mutate the incoming argv with defaults etc export const ensure = diff --git a/packages/cli/test/deploy/deploy.test.ts b/packages/cli/test/deploy/deploy.test.ts index 17c17d0fb..7a620dcc8 100644 --- a/packages/cli/test/deploy/deploy.test.ts +++ b/packages/cli/test/deploy/deploy.test.ts @@ -1,279 +1,103 @@ // Test the actual functionality of docgen // ie, generate docs to a mock folder import test from 'ava'; -import { readFileSync, existsSync } from 'node:fs'; -import fs from 'node:fs/promises'; +import { readFileSync } from 'node:fs'; import mockfs from 'mock-fs'; -import { createMockLogger } from '@openfn/logger'; -import deployHandler, { DeployFn, ensurePath } from '../../src/deploy/handler'; +import { Logger, createMockLogger } from '@openfn/logger'; +import deployHandler, { DeployFn } from '../../src/deploy/handler'; + +import { DeployError, type DeployConfig } from '@openfn/deploy'; +import { DeployOptions } from '../../src/deploy/command'; const logger = createMockLogger(); -const REPO_PATH = '/tmp/repo'; -const DOCS_PATH = `${REPO_PATH}/docs`; +const originalEnv = process.env; test.beforeEach(() => { mockfs.restore(); logger._reset(); mockfs({ - [DOCS_PATH]: {}, - }); -}); - -const loadJSON = async (path: string) => { - try { - const result = await fs.readFile(path, 'utf8'); - if (result) { - return JSON.parse(result); - } - } catch (e) {} -}; - -// Mock doc gen function -const mockGen: DeployFn = async () => ({ - name: 'test', - version: '1.0.0', - functions: [ - { - name: 'fn', - description: 'a function', - isOperation: true, - magic: false, - parameters: [], - examples: [], - }, - ], -}); - -const specifier = 'test@1.0.0'; - -const options = { - specifier, - repoDir: REPO_PATH, -}; - -test.serial('generate mock docs', async (t) => { - const path = await deployHandler(options, logger, mockGen); - t.is(path, `${DOCS_PATH}/${specifier}.json`); - - const docs = await loadJSON(path); - - t.is(docs.name, 'test'); - t.is(docs.version, '1.0.0'); -}); - -test.serial('log the result path', async (t) => { - await deployHandler(options, logger, mockGen); - - const { message } = logger._parse(logger._last); - t.is(message, ` ${DOCS_PATH}/${specifier}.json`); -}); - -test.serial("ensurePath if there's no repo", (t) => { - mockfs({ - ['/tmp']: {}, - }); - ensurePath('/tmp/repo/docs/x.json'); - - t.true(existsSync('/tmp/repo/docs')); -}); - -test.serial("ensurePath if there's no docs folder", (t) => { - mockfs({ - ['/tmp/repo']: {}, - }); - ensurePath('/tmp/repo/docs/x.json'); - - t.true(existsSync('/tmp/repo/docs')); -}); - -test.serial("ensurePath if there's a namespace", (t) => { - mockfs({ - ['/tmp']: {}, + ['./config.json']: `{"apiKey": "123"}`, + ['./project.yaml']: `{"apiKey": "123"}`, }); - ensurePath('/tmp/repo/docs/@openfn/language-common.json'); - t.true(existsSync('/tmp/repo/docs/@openfn')); + process.env = originalEnv; }); -test.serial('do not generate docs if they already exist', async (t) => { - let docgenCalled = false; +const exampleProject = readFileSync('./project.yaml', 'utf8'); - const docgen = (() => { - docgenCalled = true; - return {}; - }) as unknown as DeployFn; +type Fn = ( + ...args: Params +) => Result; - mockfs({ - [`${DOCS_PATH}/${specifier}.json`]: '{ "name": "test" }', - }); +const mockDeploy: Fn, Promise> = ( + config: DeployConfig, + _logger: Logger +) => { + return Promise.resolve(config); +}; - const path = await deployHandler(options, logger, docgen); +const options: DeployOptions = { + configPath: './config.json', + projectPath: './project.yaml', + statePath: './state.json', + command: 'deploy', + log: ['info'], + logJson: false, + confirm: false, +}; - t.false(docgenCalled); - t.is(path, `${DOCS_PATH}/${specifier}.json`); +test.serial('reads in config file', async (t) => { + await deployHandler(options, logger, mockDeploy); + t.pass(); }); -test.serial('create a placeholder before generating the docs', async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; - - // a placeholder should not exist when we start - const empty = await loadJSON(path); - t.falsy(empty); - - const docgen = (async () => { - // When docgen is called, a placeholder should now exist - const placeholder = await loadJSON(path); - t.truthy(placeholder); - t.true(placeholder.loading); - t.assert(typeof placeholder.timestamp === 'number'); - - return {}; - }) as unknown as DeployFn; +test.serial('uses confirm option for requireConfirmation', async (t) => { + let config = await deployHandler(options, logger, mockDeploy); - await deployHandler(options, logger, docgen); + t.is(config.requireConfirmation, false); }); test.serial( - 'synchronously create a placeholder before generating the docs', + 'accepts env variables to override endpoint and api key', async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; + process.env['OPENFN_API_KEY'] = 'newkey'; + let config = await deployHandler(options, logger, mockDeploy); - // a placeholder should not exist when we start - const empty = await loadJSON(path); - t.falsy(empty); + t.is(config.apiKey, 'newkey'); + t.is(config.endpoint, 'https://app.openfn.org/api/provision'); - const promise = deployHandler(options, logger, mockGen); - // the placeholder should already be created + process.env['OPENFN_ENDPOINT'] = 'http://other-endpoint.com'; + config = await deployHandler(options, logger, mockDeploy); - const placeholder = JSON.parse(readFileSync(path, 'utf8')); - t.truthy(placeholder); - t.true(placeholder.loading); - t.assert(typeof placeholder.timestamp === 'number'); - - // politely wait for the promise to run - await promise.then(); + t.is(config.apiKey, 'newkey'); + t.is(config.endpoint, 'http://other-endpoint.com'); } ); -test.serial("remove the placeholder if there's an error", async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; - - const docgen = (async () => { - // When docgen is called, a placeholder should now exist - const placeholder = await loadJSON(path); - t.truthy(placeholder); - t.true(placeholder.loading); - - throw new Error('test'); - }) as unknown as DeployFn; +test.serial('sets the exit code to 0', async (t) => { + const origExitCode = process.exitCode; + await deployHandler(options, logger, () => Promise.resolve(true)); - await deployHandler(options, logger, docgen); - - // placeholder should be gone - const empty = await loadJSON(path); - t.falsy(empty); + t.is(process.exitCode, 0); + process.exitCode = origExitCode; }); -test.serial('wait for docs if a placeholder is present', async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; - - mockfs({ - [path]: `{ "loading": true, "timestamp": ${Date.now()} }`, - }); - - // After 100ms generate some docs and write to the file - setTimeout(async () => { - const docs = await mockGen('x'); - fs.writeFile(path, JSON.stringify(docs)); - }, 60); - - let docgencalled = false; +test.serial('sets the exit code to 1', async (t) => { + const origExitCode = process.exitCode; + await deployHandler(options, logger, () => Promise.resolve(false)); - const docgen = (async () => { - docgencalled = true; - return {}; - }) as unknown as DeployFn; - - await deployHandler(options, logger, docgen, 20); - - // It should not call out to this docgen function - t.false(docgencalled); - - // docs should be present and correct - const docs = await loadJSON(path); - - t.is(docs.name, 'test'); - t.is(docs.version, '1.0.0'); + t.is(process.exitCode, 1); + process.exitCode = origExitCode; }); -test.serial("throw there's a timeout", async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; +test.serial('catches DeployErrors', async (t) => { + const origExitCode = process.exitCode; - mockfs({ - [path]: `{ "loading": true, "timestamp": ${Date.now()} }`, - }); - - // This will timeout - const timeout = 2; - await t.throwsAsync( - async () => deployHandler(options, logger, async () => {}, timeout), - { - message: 'Timed out waiting for docs to load', - } + await deployHandler(options, logger, () => + Promise.reject(new DeployError('foo bar', 'STATE_ERROR')) ); -}); - -test.serial("don't remove the placeholder if there's a timeout", async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; - - mockfs({ - [path]: `{ "loading": true, "timestamp": ${Date.now()} }`, - }); - - const timeout = 2; - await t.throwsAsync( - async () => deployHandler(options, logger, async () => {}, timeout), - { - message: 'Timed out waiting for docs to load', - } - ); - - // docs should be present and correct - const placeholder = await loadJSON(path); - - t.truthy(placeholder); - t.true(placeholder.loading); -}); - -test.serial("reset the placeholder if it's old", async (t) => { - const path = `${DOCS_PATH}/${specifier}.json`; - - mockfs({ - [path]: `{ "loading": true, "test", true, "timestamp": ${ - Date.now() - 1001 - } }`, - }); - - let docgencalled = false; - const docgen = (async (specifier: string) => { - // a new timestamp should be generated - const placeholder = await loadJSON(path); - t.true(placeholder.loading); - t.falsy(placeholder.test); - - docgencalled = true; - return await mockGen(specifier); - }) as DeployFn; - - await deployHandler(options, logger, docgen); - - // It should call out to the docgen function - t.true(docgencalled); - - // docs should be present and correct - const docs = await loadJSON(path); - t.is(docs.name, 'test'); - t.is(docs.version, '1.0.0'); + t.is(process.exitCode, 10); + process.exitCode = origExitCode; }); diff --git a/packages/cli/test/deploy/options.test.ts b/packages/cli/test/deploy/options.test.ts index 31ea1d5d7..7c5710af8 100644 --- a/packages/cli/test/deploy/options.test.ts +++ b/packages/cli/test/deploy/options.test.ts @@ -1,8 +1,8 @@ import test from 'ava'; import yargs from 'yargs'; -import deploy, { DeployOptions } from '../../src/deploy/command'; +import deployCommand, { DeployOptions } from '../../src/deploy/command'; -const cmd = yargs().command(deploy as any); +const cmd = yargs().command(deployCommand as any); const parse = (command: string) => cmd.parse(command) as yargs.Arguments; @@ -11,7 +11,7 @@ test('correct default options', (t) => { const options = parse('deploy'); t.is(options.command, 'deploy'); - t.is(options.statePath, './state.json'); + t.is(options.statePath, './.state.json'); t.is(options.projectPath, './project.yaml'); t.is(options.configPath, './.config.json'); t.falsy(options.logJson); // TODO this is undefined right now @@ -33,6 +33,6 @@ test('pass a config path (longform)', (t) => { }); test('pass a config path (shortform)', (t) => { - const options = parse('deploy -s other_config.json'); + const options = parse('deploy -c other_config.json'); t.deepEqual(options.configPath, 'other_config.json'); }); diff --git a/packages/cli/tsconfig.json b/packages/cli/tsconfig.json index 8906c56a5..98b743ebf 100644 --- a/packages/cli/tsconfig.json +++ b/packages/cli/tsconfig.json @@ -2,6 +2,9 @@ "extends": "../../tsconfig.common", "include": ["src/**/*.ts", "test/**/*.ts"], "compilerOptions": { - "module": "ESNext" + "module": "ESNext", + "paths": { + "@openfn/*": ["../*"] + } } } diff --git a/packages/deploy/src/client.ts b/packages/deploy/src/client.ts index b554e706c..cd74408f7 100644 --- a/packages/deploy/src/client.ts +++ b/packages/deploy/src/client.ts @@ -1,42 +1,100 @@ -import { DeployOptions } from './types'; - -export async function getProject(config: DeployOptions, projectId: string) { - const response = await fetch(`${config.endpoint}/projects/${projectId}`, { - headers: { - Authorization: `Bearer ${config.apiKey}`, - Accept: 'application/json', - }, - }); - - // A 404 response means the project doesn't exist yet. - if (response.status === 404) { - return null; - } +import { DeployConfig, ProjectPayload } from './types'; +import { DeployError } from './deployError'; - if (!response.ok) { - throw new Error( - `Failed to fetch project ${projectId}: ${response.statusText}` - ); +export async function getProject( + config: DeployConfig, + projectId: string +): Promise<{ data: ProjectPayload | null }> { + const url = new URL(config.endpoint + `/${projectId}`); + + try { + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${config.apiKey}`, + Accept: 'application/json', + }, + }); + + // A 404 response means the project doesn't exist yet. + if (response.status === 404) { + return { data: null }; + } + + if (!response.ok) { + handle401(config, response); + + throw new Error( + `Failed to fetch project ${projectId}: ${response.statusText}` + ); + } + + return response.json(); + } catch (error: any) { + handleCommonErrors(config, error); + + throw error; } +} + +export async function deployProject( + config: DeployConfig, + payload: any +): Promise<{ data: ProjectPayload }> { + try { + const url = new URL(config.endpoint); + const response = await fetch(url, { + method: 'POST', + headers: { + Authorization: `Bearer ${config.apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }); - return response.json(); + if (!response.ok) { + if (response.status === 422) { + const body = await response.json(); + + throw new DeployError( + `Failed to deploy project ${payload.name}:\n${JSON.stringify( + body, + null, + 2 + )}`, + 'DEPLOY_ERROR' + ); + } + + handle401(config, response); + + throw new DeployError( + `Failed to deploy project ${payload.name}: ${response.statusText}`, + 'DEPLOY_ERROR' + ); + } + + return response.json(); + } catch (error: any) { + handleCommonErrors(config, error); + + throw error; + } } -export async function deployProject(config: DeployOptions, payload: any) { - const response = await fetch(`${config.endpoint}/projects`, { - method: 'POST', - headers: { - Authorization: `Bearer ${config.apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(payload), - }); - - if (!response.ok) { - throw new Error( - `Failed to deploy project ${payload.name}: ${response.statusText}` +function handle401(config, response: Response) { + if (response.status === 401) { + throw new DeployError( + `Failed to authorize request with endpoint ${config.endpoint}, got 401 Unauthorized.`, + 'DEPLOY_ERROR' ); } +} - return response.json(); +function handleCommonErrors(config, error: any) { + if (error.cause?.code === 'ECONNREFUSED') { + throw new DeployError( + `Failed to connect to endpoint ${config.endpoint}, got ECONNREFUSED.`, + 'DEPLOY_ERROR' + ); + } } diff --git a/packages/deploy/src/deployError.ts b/packages/deploy/src/deployError.ts new file mode 100644 index 000000000..00b9a2ef6 --- /dev/null +++ b/packages/deploy/src/deployError.ts @@ -0,0 +1,16 @@ +type DeployErrorName = + | 'VALIDATION_ERROR' + | 'DEPLOY_ERROR' + | 'CONFIG_ERROR' + | 'STATE_ERROR' + | 'SPEC_ERROR'; + +export class DeployError extends Error { + name: DeployErrorName; + + constructor(message: string, name: DeployErrorName) { + super(message); + Object.setPrototypeOf(this, DeployError.prototype); + this.name = name; + } +} diff --git a/packages/deploy/src/index.ts b/packages/deploy/src/index.ts index ddb2884a1..69edc1ccf 100644 --- a/packages/deploy/src/index.ts +++ b/packages/deploy/src/index.ts @@ -1,7 +1,8 @@ import { confirm } from '@inquirer/prompts'; -import { DeployOptions, ProjectState } from './types'; +import { inspect } from 'node:util'; +import { DeployConfig, ProjectState } from './types'; import { readFile, writeFile } from 'fs/promises'; -import { validate } from './validator'; +import { parseAndValidate } from './validator'; import jsondiff from 'json-diff'; import { mergeProjectPayloadIntoState, @@ -9,39 +10,84 @@ import { toProjectPayload, } from './stateTransform'; import { deployProject, getProject } from './client'; +import { DeployError } from './deployError'; +import { Logger } from '@openfn/logger'; // =============== Configuration =============== -const defaultOptions: DeployOptions = { - apiKey: null, - specPath: 'project.yaml', - statePath: '.state.json', - endpoint: 'https://app.openfn.org/api/provision', - requireConfirmation: true, - dryRun: false, -}; - -async function getConfig(path?: string): Promise { - // TODO: merge env vars into either defaultOptions or config file results - // TODO: validate config after merging/reading +function mergeDefaultOptions(options: Partial): DeployConfig { + return { + apiKey: null, + configPath: '.config.json', + specPath: 'project.yaml', + statePath: '.state.json', + endpoint: 'https://app.openfn.org/api/provision', + requireConfirmation: true, + dryRun: false, + ...options, + }; +} + +export async function getConfig(path?: string): Promise { try { - const config = await readFile(path ?? '.config.json', 'utf8'); - return JSON.parse(config); + return mergeDefaultOptions( + JSON.parse(await readFile(path ?? '.config.json', 'utf8')) + ); } catch (error) { - return defaultOptions; + return mergeDefaultOptions({}); } } -function validateConfig(config: DeployOptions) { +export function validateConfig(config: DeployConfig) { if (!config.apiKey) { - throw new Error('Missing API key'); + throw new DeployError('Missing API key', 'CONFIG_ERROR'); } try { new URL(config.endpoint); - } catch (error) { + } catch (error: any) { if (error.code === 'ERR_INVALID_URL') { - throw new Error('Invalid endpoint'); + throw new DeployError('Invalid endpoint', 'CONFIG_ERROR'); + } else { + throw error; + } + } +} + +// =================== State =================== + +async function readState(path: string) { + const state = await readFile(path, 'utf8'); + + return JSON.parse(state) as ProjectState; +} + +async function getState(path: string) { + try { + return await readState(path); + } catch (error: any) { + if (error.code === 'ENOENT') { + return { workflows: {} } as ProjectState; + } else { + throw error; + } + } +} + +function writeState(config: DeployConfig, nextState: {}): Promise { + return writeFile(config.statePath, JSON.stringify(nextState, null, 2)); +} + +// ==================== Spec =================== + +// Given a path to a project spec, read and validate it. +async function getSpec(path: string) { + try { + const body = await readFile(path, 'utf8'); + return parseAndValidate(body); + } catch (error: any) { + if (error.code === 'ENOENT') { + throw new DeployError(`File not found: ${path}`, 'SPEC_ERROR'); } else { throw error; } @@ -50,66 +96,77 @@ function validateConfig(config: DeployOptions) { // ============================================= -export async function deploy(config: DeployOptions) { +export async function deploy(config: DeployConfig, logger: Logger) { const [state, spec] = await Promise.all([ - readState(config.statePath), - readSpec(config.specPath), + getState(config.statePath), + getSpec(config.specPath), ]); + logger.debug('spec', spec); + if (spec.errors.length > 0) { + spec.errors.forEach((e) => logger.warn(e.message)); + throw new DeployError(`${config.specPath} has errors`, 'VALIDATION_ERROR'); + } const nextState = mergeSpecIntoState(state, spec.doc); validateProjectState(nextState); // Convert the state to a payload for the API. const nextProject = toProjectPayload(nextState); - const currentProject = await getProject(config, nextState.id); - renderDiff(currentProject, nextProject); + logger.debug('Getting project from server...'); + const { data: currentProject } = await getProject(config, nextState.id); + + logger.debug( + 'currentProject', + '\n' + inspect(currentProject, { colors: true }) + ); + logger.debug('nextProject', '\n' + inspect(nextProject, { colors: true })); + + const diff = jsondiff.diffString(currentProject, nextProject); + + if (!diff) { + logger.log('No changes to deploy.'); + return true; + } + + logger.info(diff); if (config.dryRun) { - return; + return true; } if (config.requireConfirmation) { - if (!(await confirm({ message: 'Deploy?' }))) { - console.log('Aborting.'); - return; + if (!(await confirm({ message: 'Deploy?', default: false }))) { + logger.log('Cancelled.'); + return false; } } - const deployedProject = await deployProject(config, nextProject); + const { data: deployedProject } = await deployProject(config, nextProject); - const deployedState = mergeProjectPayloadIntoState(nextState, deployedProject); + logger.debug('deployedProject', deployedProject); + const deployedState = mergeProjectPayloadIntoState( + nextState, + deployedProject + ); // IDEA: perhaps before writing, we should check if the current project.yaml // merges into the deployed state to produce the current state. If not, we // should warn the user that the project.yaml is out of sync with the server? await writeState(config, deployedState); + return true; } function validateProjectState(state: ProjectState) { if (!state.workflows) { - throw new Error('Project state must have a workflows property'); + throw new DeployError( + 'Project state must have a workflows property', + 'STATE_ERROR' + ); } } -// Given a path to a project spec, read and validate it. -async function readSpec(path: string) { - const body = await readFile(path, 'utf8'); - return validate(body); -} - -async function readState(path: string) { - const state = await readFile(path, 'utf8'); - - return JSON.parse(state) as ProjectState; -} - -function renderDiff(before: {}, after: {}) { - console.log(jsondiff.diffString(before, after)); -} - -function writeState(config: DeployOptions, nextState: {}): Promise { - return writeFile(config.statePath, JSON.stringify(nextState, null, 2)); -} +export type { DeployConfig, ProjectState }; +export { DeployError }; diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index c2885d50a..b0b3c84fb 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -8,13 +8,16 @@ import { StateEdge, WorkflowSpec, WorkflowState, - Job, } from './types'; import { isEmpty, pickKeys, splitZip } from './utils'; +import { DeployError } from './deployError'; -function mergeJobs(stateJobs, specJobs): WorkflowState['jobs'] { +function mergeJobs( + stateJobs: WorkflowState['jobs'], + specJobs: WorkflowSpec['jobs'] +): WorkflowState['jobs'] { return Object.fromEntries( - splitZip(stateJobs, specJobs).map(([jobKey, stateJob, specJob]) => { + splitZip(stateJobs, specJobs || {}).map(([jobKey, stateJob, specJob]) => { if (specJob && !stateJob) { return [ jobKey, @@ -23,7 +26,7 @@ function mergeJobs(stateJobs, specJobs): WorkflowState['jobs'] { name: specJob.name, adaptor: specJob.adaptor, body: specJob.body, - enabled: pickValue(specJob, stateJob, 'enabled', true), + enabled: pickValue(specJob, stateJob || {}, 'enabled', true), }, ]; } @@ -32,16 +35,25 @@ function mergeJobs(stateJobs, specJobs): WorkflowState['jobs'] { return [jobKey, { id: stateJob.id, delete: true }]; } - return [ - jobKey, - { - id: stateJob.id, - name: specJob.name, - adaptor: specJob.adaptor, - body: specJob.body, - enabled: pickValue(specJob, stateJob, 'enabled', true), - }, - ]; + if (specJob && stateJob) { + return [ + jobKey, + { + id: stateJob.id, + name: specJob.name, + adaptor: specJob.adaptor, + body: specJob.body, + enabled: pickValue(specJob, stateJob, 'enabled', true), + }, + ]; + } + + throw new DeployError( + `Invalid job spec or corrupted state for job with key: ${String( + jobKey + )}`, + 'VALIDATION_ERROR' + ); }) ); } @@ -138,8 +150,8 @@ function mergeEdges( return [ edgeKey, { - id: crypto.randomUUID(), ...convertToStateEdge(jobs, triggers, specEdge), + id: crypto.randomUUID(), }, ]; } @@ -151,8 +163,8 @@ function mergeEdges( return [ edgeKey, { - id: stateEdge.id, - ...convertToStateEdge(jobs, triggers, specEdge), + ...convertToStateEdge(jobs, triggers, specEdge!), + id: stateEdge!.id, }, ]; } @@ -202,6 +214,7 @@ export function mergeSpecIntoState( return [ workflowKey, { + ...stateWorkflow, id: stateWorkflow.id, name: specWorkflow.name, jobs: nextJobs, @@ -214,6 +227,7 @@ export function mergeSpecIntoState( ); return { + ...oldState, id: oldState.id || crypto.randomUUID(), name: spec.name, workflows: nextWorkflows, @@ -257,8 +271,7 @@ export function mergeProjectPayloadIntoState( ); return { - id: project.id, - name: project.name, + ...project, workflows: nextWorkflows, }; } @@ -288,8 +301,7 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { state.workflows ).map((workflow) => { return { - id: workflow.id, - name: workflow.name, + ...workflow, jobs: Object.values( workflow.jobs ) as ProjectPayload['workflows'][0]['jobs'], @@ -303,8 +315,7 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { }); return { - id: state.id, - name: state.name, + ...state, workflows, }; } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 31b371a86..3887a2db5 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -69,7 +69,8 @@ export interface ProjectPayload { type Concrete = Type & { id: string }; -export interface DeployOptions { +export interface DeployConfig { + configPath?: string; specPath: string; statePath: string; endpoint: string; diff --git a/packages/deploy/src/utils.ts b/packages/deploy/src/utils.ts index b69e8c4a8..aac21dfda 100644 --- a/packages/deploy/src/utils.ts +++ b/packages/deploy/src/utils.ts @@ -10,7 +10,7 @@ export const uuidRegex = new RegExp( export function splitZip< L extends { [k: string]: any }, R extends { [k: string]: any } ->(l: L, r: R): [string, L | null, R | null][] { +>(l: L, r: R): [keyof L | keyof R, L[keyof L] | null, R[keyof R] | null][] { return concatKeys(l, r).map((key) => { return [key, l[key], r[key]]; }); @@ -29,11 +29,11 @@ export function mapReduce( } // Get all keys from all objects and ensure they are unique -export function concatKeys(...objs: T[]) { +export function concatKeys(...objs: Object[]): string[] { return objs - .reduce((acc, obj) => { + .reduce((acc, obj) => { return acc.concat(Object.keys(obj)); - }, [] as string[]) + }, []) .reduce((acc, key) => { if (!acc.includes(key)) { acc.push(key); diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index f9eddcf27..730443030 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -12,8 +12,16 @@ interface Error { path?: string[]; } -export function validate(input: string): { errors: Error[]; doc: Project } { - let errors = []; +export function parseAndValidate(input: string): { + errors: Error[]; + doc: Project; +} { + let errors: { + context: any; + message: string; + path?: string[]; + range: [number, number, number]; + }[] = []; let keys: string[] = []; const doc = YAML.parseDocument(input); @@ -65,12 +73,38 @@ export function validate(input: string): { errors: Error[]; doc: Project } { } } + YAML.visit(doc, { + Pair(_, pair) { + if (pair.key && pair.key.value === 'workflows') { + if (pair.value.value === null) { + errors.push({ + context: pair, + message: 'project: must provide at least one workflow', + path: ['workflows'], + }); + + return doc.createPair('workflows', {}); + } + } + + if (pair.key && pair.key.value === 'jobs') { + if (pair.value.value === null) { + errors.push({ + context: pair, + message: 'jobs: must be a map', + range: pair.value.range, + }); + + return doc.createPair('jobs', {}); + } + } + }, + }); + if (!doc.has('name')) { errors.push({ context: doc, message: 'Project must have a name' }); } - // console.log(doc); - const workflows = doc.getIn(['workflows']); validateWorkflows(workflows); diff --git a/packages/deploy/test/changeFormatter.test.ts b/packages/deploy/test/changeFormatter.test.ts deleted file mode 100644 index 03ca8acd8..000000000 --- a/packages/deploy/test/changeFormatter.test.ts +++ /dev/null @@ -1,65 +0,0 @@ -import test from 'ava'; -import { diffString, diff } from 'json-diff'; -import { fullExampleState } from './fixtures'; -import jsonpatch from 'fast-json-patch'; - -const newProjectPatches = [ - { - op: 'add', - path: '/id', - value: crypto.randomUUID(), - }, - { - op: 'add', - path: '/name', - value: 'project-name', - }, - { - op: 'add', - path: '/workflows/workflow-one', - value: { - id: crypto.randomUUID(), - name: 'workflow one', - jobs: { - 'new-job': { - name: 'new job', - adaptor: '@openfn/language-adaptor', - body: 'foo()', - }, - }, - triggers: { - 'trigger-one': { - type: 'cron', - }, - }, - edges: {}, - }, - }, - ] - -test('can render a pretty diff for a given patch', (t) => { - const state = fullExampleState(); - const patches = [ - { - op: 'add', - path: '/workflows/workflow-one/edges/job-a->job-b/delete', - value: true, - }, - ]; - - const patchedState = jsonpatch.applyPatch( - state, - patches, - false, - false - ).newDocument; - - // console.log(JSON.stringify(patchedState, null, 2)); - - console.log(diffString(state, patchedState)); - - t.pass(); -}); - - -test("can render a set of text diffs for a set of path") diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index 1e3282e8e..cd170c9bf 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -1,4 +1,4 @@ -import { ProjectSpec } from "../src/types"; +import { ProjectSpec } from '../src/types'; export function fullExampleSpec() { return { diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index f5a5b7fd0..82e1d03e9 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -1,5 +1,4 @@ import test from 'ava'; -import crypto from 'crypto'; import jp from 'jsonpath'; import { mergeProjectPayloadIntoState, @@ -81,7 +80,7 @@ test('toNextState adding a job', (t) => { }); }); -function getItem(result, itemType, key) { +function getItem(result: {}, itemType: string, key: string) { const items = jp.query(result, `$..workflows[*].${itemType}["${key}"]`); if (items.length === 0) { @@ -371,13 +370,9 @@ test('mergeProjectIntoState with deletions', (t) => { { id: workflowOne.id, name: workflowOne.name, - jobs: [ - getItem(existingState, 'jobs', 'job-a'), - ], + jobs: [getItem(existingState, 'jobs', 'job-a')], triggers: [getItem(existingState, 'triggers', 'trigger-one')], - edges: [ - getItem(existingState, 'edges', 'trigger-one->job-a'), - ], + edges: [getItem(existingState, 'edges', 'trigger-one->job-a')], }, ], }; diff --git a/packages/deploy/test/validator.test.ts b/packages/deploy/test/validator.test.ts index 2af824465..1eb29c66b 100644 --- a/packages/deploy/test/validator.test.ts +++ b/packages/deploy/test/validator.test.ts @@ -1,5 +1,5 @@ import test from 'ava'; -import { validate } from '../src/validator'; +import { parseAndValidate } from '../src/validator'; function findError(errors: any[], message: string) { return errors.find((e) => e.message === message); @@ -13,7 +13,7 @@ workflows: - name: workflow-two `; - let results = validate(doc); + let results = parseAndValidate(doc); t.truthy( results.errors.find((e) => e.message === 'workflows: must be a map') @@ -38,7 +38,7 @@ workflows: - 2 `; - results = validate(doc); + results = parseAndValidate(doc); t.truthy(findError(results.errors, 'duplicate key: workflow-one')); diff --git a/packages/deploy/tsconfig.json b/packages/deploy/tsconfig.json index 4c67ff938..35f121ed6 100644 --- a/packages/deploy/tsconfig.json +++ b/packages/deploy/tsconfig.json @@ -1,10 +1,7 @@ { "compilerOptions": { - "target": "es2020", + "target": "es2020" }, "extends": "../../tsconfig.common", - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] + "include": ["src/**/*.ts", "test/**/*.ts"] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 626622286..b4fb52d78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: '@openfn/compiler': specifier: workspace:* version: link:../compiler + '@openfn/deploy': + specifier: workspace:* + version: link:../deploy '@openfn/describe-package': specifier: workspace:* version: link:../describe-package @@ -4834,7 +4837,7 @@ packages: optional: true dependencies: lilconfig: 2.0.6 - ts-node: 10.9.1(@types/node@18.7.18)(typescript@4.8.3) + ts-node: 10.9.1(@types/node@17.0.45)(typescript@4.8.3) yaml: 1.10.2 dev: true