Skip to content

Commit

Permalink
cli: use cached input state if appropriate
Browse files Browse the repository at this point in the history
  • Loading branch information
josephjclark committed Apr 2, 2024
1 parent d6f5a8f commit e2a56cc
Show file tree
Hide file tree
Showing 3 changed files with 34 additions and 7 deletions.
2 changes: 1 addition & 1 deletion packages/cli/src/execute/handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ const executeHandler = async (options: ExecuteOptions, logger: Logger) => {
}
}

const state = await loadState(options, logger);
const state = await loadState(plan, options, logger);

if (options.compile) {
plan = (await compile(plan, options, logger)) as ExecutionPlan;
Expand Down
11 changes: 6 additions & 5 deletions packages/cli/src/util/cache.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,27 @@
import { ExecutionPlan } from '@openfn/lexicon';
import fs from 'node:fs/promises';
import path from 'node:path'

import type { Opts } from '../options';

export const getCachePath = async (plan, options, stepId) => {
export const getCachePath = async (plan: ExecutionPlan, options: Pick<Opts, 'baseDir' | 'cache'>, stepId: string) => {
const { baseDir } = options;

// TODO ensure that plan always has a name
// ( I do think the CLI sets a default?)
const { name } = plan.workflow;

const basePath = `${baseDir}/.cli-cache/${name}`;
await fs.mkdir(basePath, { recursive: true })
if (stepId) {
const step = plan.workflow.steps.find(({ id }) => id === stepId);
return path.resolve(`${basePath}/${step.name.replace(/ /, '-')}.json`);
const fileName = step?.name ?? stepId;
return path.resolve(`${basePath}/${fileName.replace(/ /, '-')}.json`);
}
return path.resolve(basePath);

}

// TODO this needs to move out into a util or something
export const saveToCache = async (plan, stepId, output, options) => {
export const saveToCache = async (plan: ExecutionPlan, stepId: string, output: any, options: Pick<Opts, 'baseDir' | 'cache'>) => {
if (options.cache) {
const cachePath = await getCachePath(plan, options, stepId);
// TODO use the CLI logger
Expand Down
28 changes: 27 additions & 1 deletion packages/cli/src/util/load-state.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
import fs from 'node:fs/promises';

import { getCachePath } from './cache';

import type { ExecutionPlan } from '@openfn/lexicon';
import type { Logger } from '@openfn/logger';
import type { Opts } from '../options';

export default async (
opts: Pick<Opts, 'stateStdin' | 'statePath'>,
plan: ExecutionPlan,
opts: Pick<Opts, 'baseDir' | 'stateStdin' | 'statePath' | 'cache' | 'start'>,
log: Logger
) => {
const { stateStdin, statePath } = opts;
Expand Down Expand Up @@ -35,6 +40,27 @@ export default async (
}
}

if (opts.cache && opts.start) {
log.info(`Attempting to load cached state for step "${opts.start}"`)
try {
const cachedStatePath = await getCachePath(plan, opts, opts.start)
log.debug('Loading cached state from', cachedStatePath)

const exists = await fs.lstat(cachedStatePath)
if (exists) {
const str = await fs.readFile(cachedStatePath, 'utf8');
const json = JSON.parse(str);
log.success(`Loaded cached state for step "${opts.start}"`)
log.debug('state:', json);
} else {
log.warn(`No cached state found for step "${opts.start}"`);
log.warn('Re-run this command with --cache and without --start to rebuild the cache');
}
} catch(e) {
log.warn('Error loading cached state')
}
}

log.info(
'No state provided - using default state { data: {}, configuration: {} }'
);
Expand Down

0 comments on commit e2a56cc

Please sign in to comment.