From 980f8ad90aefb10852c2498b8b3e24c89f246ad2 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 5 Dec 2024 16:10:31 +0000
Subject: [PATCH 01/23] Added a fairly basic get handler

---
 packages/cli/package.json               |   1 +
 packages/cli/src/cli.ts                 |   2 +
 packages/cli/src/collections/command.ts | 116 ++++++
 packages/cli/src/collections/handler.ts |  43 +++
 packages/cli/src/collections/request.ts | 117 ++++++
 packages/cli/src/commands.ts            |  10 +-
 pnpm-lock.yaml                          | 458 ++++++++++++++++++++++--
 7 files changed, 726 insertions(+), 21 deletions(-)
 create mode 100644 packages/cli/src/collections/command.ts
 create mode 100644 packages/cli/src/collections/handler.ts
 create mode 100644 packages/cli/src/collections/request.ts

diff --git a/packages/cli/package.json b/packages/cli/package.json
index 5d85d7415..e2bce04c3 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -59,6 +59,7 @@
     "figures": "^5.0.0",
     "rimraf": "^3.0.2",
     "treeify": "^1.1.0",
+    "undici": "^7.1.0",
     "ws": "^8.18.0",
     "yargs": "^17.7.2"
   },
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
index 9a500a4f0..4b180de6a 100644
--- a/packages/cli/src/cli.ts
+++ b/packages/cli/src/cli.ts
@@ -2,6 +2,7 @@ import yargs, { Arguments } from 'yargs';
 import { hideBin } from 'yargs/helpers';
 
 import apolloCommand from './apollo/command';
+import collectionsCommand from './collections/command';
 import compileCommand from './compile/command';
 import deployCommand from './deploy/command';
 import docgenCommand from './docgen/command';
@@ -19,6 +20,7 @@ export const cmd = y
   // TODO Typescipt hacks because signatures don't seem to align
   .command(executeCommand as any)
   .command(compileCommand)
+  .command(collectionsCommand)
   .command(deployCommand as any)
   .command(installCommand) // allow install to run from the top as well as repo
   .command(repoCommand)
diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
new file mode 100644
index 000000000..79456d1a5
--- /dev/null
+++ b/packages/cli/src/collections/command.ts
@@ -0,0 +1,116 @@
+import yargs from 'yargs';
+import * as o from '../options';
+import type { Opts } from '../options';
+import { build, ensure, override } from '../util/command-builders';
+
+export type CollectionsOptions = Pick<
+  Opts,
+  'log' | 'logJson' | 'outputPath' | 'outputStdout'
+> & {
+  lightning?: string;
+  token?: string;
+  pageSize?: number;
+  limit?: number;
+  key: string;
+  collectionName: string;
+  pretty?: boolean;
+};
+
+const desc = `Call out to the Collections API on Lightning`;
+
+export default {
+  command: 'collections [subcommand]',
+  describe: desc,
+  builder: (yargs) =>
+    yargs
+      .command(get)
+      .example(
+        'collections get my-collection 2024* -O',
+        'Get all keys from my-collection starting with the string "2024" and log the results to stdout'
+      ),
+} as yargs.CommandModule<{}>;
+
+// Since these options only apply to collections,
+// Let's not declare them centrally, but keep them here
+const collectionName = {
+  name: 'collection-name',
+  yargs: {
+    alias: ['name'],
+    description: 'Name of the collection to fetch from',
+    demand: true,
+  },
+};
+
+const key = {
+  name: 'key',
+  yargs: {
+    description: 'Key or key pattern to retrieve',
+    demand: true,
+  },
+};
+
+// TODO this should default from env
+// TODO this is used by other args
+const token = {
+  name: 'pat',
+  yargs: {
+    alias: ['token'],
+    description: 'Lightning Personal Access Token (PAT)',
+  },
+};
+
+const lightningUrl = {
+  name: 'lightning',
+  yargs: {
+    description: 'URL to Lightning server',
+  },
+};
+
+const pageSize = {
+  name: 'page-size',
+  yargs: {
+    description: 'Number of items to fetch per page',
+    type: 'number',
+  },
+};
+
+// TODO not working yet
+const limit = {
+  name: 'limit',
+  yargs: {
+    description: 'Maximum number of items to download',
+    type: 'number',
+  },
+};
+
+const pretty = {
+  name: 'pretty',
+  yargs: {
+    description: 'Prettify serialized output',
+    type: 'boolean',
+  },
+};
+
+const getOptions = [
+  collectionName,
+  key,
+  token,
+  lightningUrl,
+  pageSize,
+  limit,
+  pretty,
+
+  o.stateStdin,
+  override(o.log, {
+    default: 'info',
+  }),
+  o.logJson,
+  o.outputPath,
+];
+
+export const get = {
+  command: 'get [name] [key]',
+  describe: 'Get values from a collection',
+  handler: ensure('collections-get', getOptions),
+  builder: (yargs) => build(getOptions, yargs),
+} as yargs.CommandModule<{}>;
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
new file mode 100644
index 000000000..2075eef90
--- /dev/null
+++ b/packages/cli/src/collections/handler.ts
@@ -0,0 +1,43 @@
+import { writeFile } from 'node:fs/promises';
+
+import { Logger } from '../util/logger';
+import { CollectionsOptions } from './command';
+import request from './request';
+
+export const get = async (options: CollectionsOptions, logger: Logger) => {
+  logger.info(
+    `Fetching "${options.key}" from collection "${options.collectionName}"`
+  );
+
+  const result = await request(
+    'GET',
+    {
+      lightning: options.lightning,
+      token: options.token,
+      pageSize: options.pageSize,
+      limit: options.limit,
+      key: options.key,
+      collectionName: options.collectionName,
+    },
+    logger
+  );
+
+  logger.success(`Fetched ${result.count} items!`);
+
+  if (options.outputPath) {
+    const content = JSON.stringify(
+      result,
+      null,
+      options.pretty ? 2 : undefined
+    );
+    await writeFile(options.outputPath!, content);
+    logger.always(`Wrote items to ${options.outputPath}`);
+  } else {
+    // use print because it won't stringify
+    logger.print(result.items);
+  }
+};
+
+export default {
+  get,
+};
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
new file mode 100644
index 000000000..95b78381d
--- /dev/null
+++ b/packages/cli/src/collections/request.ts
@@ -0,0 +1,117 @@
+import path from 'node:path';
+import { request } from 'undici';
+import { Logger } from '../util';
+
+// helper function to call out to the collections API
+
+// export const request = async (state, client, path, options = {}) => {
+
+type Options = {
+  key: string;
+  collectionName: string;
+  token: string;
+  lightning?: string;
+
+  includeMeta?: boolean; // TODO ignored right now
+  pageSize?: number;
+};
+
+type Key = string;
+
+// This is what uploaded and downloaded data looks like
+// Uploads don't include count and meta, but they're harmless
+// For now, meta is discarded
+type ItemSet = {
+  items: Record<Key, any>;
+  count?: number;
+  meta?: Record<
+    Key,
+    {
+      updated: string;
+      created: string;
+    }
+  >;
+};
+
+// TODO we should try to autoparse strings into json right?
+// we can take a flag to not do that
+
+// TODO how should we return data?
+// As a key: value object?
+// what about metadata?
+// Let's add that as a second object
+// so you get: { items, metadata }
+
+// TODO how should we handle cursor?
+// Lets a) support limit and b) fetch everything
+export default async (
+  method: 'GET' | 'POST',
+  options: Options,
+  logger: Logger
+) => {
+  // if (!state.configuration.collections_token) {
+  //   throwError('INVALID_AUTH', {
+  //     description: 'No access key provided for collection request',
+  //     fix: 'Ensure the "collections_token" value is set on state.configuration',
+  //     path,
+  //   });
+  // }
+
+  const base =
+    options.lightning || 'http://localhost:4000' || 'https://app.openfn.org';
+
+  const url = path.join(base, '/collections', options.collectionName);
+
+  logger.debug('Calling Collections server at ', url);
+
+  const headers = {
+    Authorization: `Bearer ${options.token}`,
+  };
+
+  const query: any = {
+    key: options.key,
+    limit: options.pageSize || 1000,
+  };
+
+  const args = {
+    headers,
+    method,
+    query,
+  };
+
+  const result: ItemSet = {
+    count: 0, // Set the count here so that it comes up first when serialized
+    items: {},
+  };
+
+  let cursor;
+  do {
+    if (cursor) {
+      args.query.cursor = cursor;
+    }
+
+    const response = await request(url, args);
+    if (response.statusCode >= 400) {
+      // await handleError(response, path, state.configuration.collections_endpoint);
+      logger.error('error!');
+    }
+    const items: any = await response.body.json();
+    logger.debug(
+      'Received',
+      response.statusCode,
+      `- ${items.items.length} values`
+    );
+    for (const item of items.items) {
+      try {
+        result.items[item.key] = JSON.parse(item.value);
+      } catch (e) {
+        result.items[item.key] = item.value;
+      }
+    }
+    cursor = items.cursor;
+  } while (cursor);
+
+  result.count = Object.keys(result.items).length;
+
+  return result;
+};
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index 9c84e9ca4..f2906a65a 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -2,6 +2,7 @@ import { Opts } from './options';
 import apollo from './apollo/handler';
 import execute from './execute/handler';
 import compile from './compile/handler';
+import collections from './collections/handler';
 import test from './test/handler';
 import deploy from './deploy/handler';
 import docgen from './docgen/handler';
@@ -19,6 +20,7 @@ import printVersions from './util/print-versions';
 export type CommandList =
   | 'apollo'
   | 'compile'
+  | 'collections-get'
   | 'deploy'
   | 'docgen'
   | 'docs'
@@ -42,6 +44,7 @@ const handlers = {
   docs,
   metadata,
   pull,
+  ['collections-get']: collections.get,
   ['repo-clean']: clean,
   ['repo-install']: install,
   ['repo-pwd']: pwd,
@@ -81,10 +84,15 @@ const parse = async (options: Opts, log?: Logger) => {
     ) as string[];
   }
 
+  // TODO Please fix this joe!
+  // Put the validation isnide the repoDir option
+
   // TODO it would be nice to do this in the repoDir option, but
   // the logger isn't available yet
   if (
-    !/^(pull|deploy|test|version|apollo)$/.test(options.command!) &&
+    !/^(pull|deploy|test|version|apollo|collections-get)$/.test(
+      options.command!
+    ) &&
     !options.repoDir
   ) {
     logger.warn(
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 90d81424e..95d2fc642 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -221,6 +221,9 @@ importers:
       treeify:
         specifier: ^1.1.0
         version: 1.1.0
+      undici:
+        specifier: ^7.1.0
+        version: 7.1.0
       ws:
         specifier: ^8.18.0
         version: 8.18.0
@@ -464,9 +467,17 @@ importers:
         specifier: ^5.1.6
         version: 5.1.6
 
-  packages/engine-multi/tmp/a/b/c: {}
+  packages/engine-multi/tmp/a/b/c:
+    dependencies:
+      ava:
+        specifier: '6'
+        version: 6.2.0
 
-  packages/engine-multi/tmp/repo: {}
+  packages/engine-multi/tmp/repo:
+    dependencies:
+      ava:
+        specifier: '6'
+        version: 6.2.0
 
   packages/lexicon:
     devDependencies:
@@ -1634,6 +1645,24 @@ packages:
       read-yaml-file: 1.1.0
     dev: true
 
+  /@mapbox/node-pre-gyp@1.0.11:
+    resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==}
+    hasBin: true
+    dependencies:
+      detect-libc: 2.0.3
+      https-proxy-agent: 5.0.1
+      make-dir: 3.1.0
+      node-fetch: 2.6.7
+      nopt: 5.0.0
+      npmlog: 5.0.1
+      rimraf: 3.0.2
+      semver: 7.6.3
+      tar: 6.2.1
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+    dev: false
+
   /@nodelib/fs.scandir@2.1.5:
     resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==}
     engines: {node: '>= 8'}
@@ -1708,6 +1737,25 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/pluginutils@5.1.3:
+    resolution: {integrity: sha512-Pnsb6f32CD2W3uCaLZIzDmeFyQ2b8UWMFI7xtwUezpcGBDVDW6y9XgAWIlARiGAo6eNF5FK5aQTr0LFyNyqq5A==}
+    engines: {node: '>=14.0.0'}
+    peerDependencies:
+      rollup: ^1.20.0||^2.0.0||^3.0.0||^4.0.0
+    peerDependenciesMeta:
+      rollup:
+        optional: true
+    dependencies:
+      '@types/estree': 1.0.6
+      estree-walker: 2.0.2
+      picomatch: 4.0.2
+    dev: false
+
+  /@sindresorhus/merge-streams@2.3.0:
+    resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==}
+    engines: {node: '>=18'}
+    dev: false
+
   /@slack/logger@3.0.0:
     resolution: {integrity: sha512-DTuBFbqu4gGfajREEMrkq5jBhcnskinhr4+AnfJEk48zhVeEv3XnUKGIX98B74kxhYsIMfApGGySTn7V3b5yBA==}
     engines: {node: '>= 12.13.0', npm: '>= 6.12.0'}
@@ -1787,6 +1835,10 @@ packages:
       '@types/keygrip': 1.0.2
       '@types/node': 18.15.13
 
+  /@types/estree@1.0.6:
+    resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==}
+    dev: false
+
   /@types/events@3.0.0:
     resolution: {integrity: sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g==}
     dev: true
@@ -2055,9 +2107,31 @@ packages:
       - supports-color
     dev: false
 
+  /@vercel/nft@0.27.7:
+    resolution: {integrity: sha512-FG6H5YkP4bdw9Ll1qhmbxuE8KwW2E/g8fJpM183fWQLeVDGqzeywMIeJ9h2txdWZ03psgWMn6QymTxaDLmdwUg==}
+    engines: {node: '>=16'}
+    hasBin: true
+    dependencies:
+      '@mapbox/node-pre-gyp': 1.0.11
+      '@rollup/pluginutils': 5.1.3
+      acorn: 8.14.0
+      acorn-import-attributes: 1.9.5(acorn@8.14.0)
+      async-sema: 3.1.1
+      bindings: 1.5.0
+      estree-walker: 2.0.2
+      glob: 7.2.3
+      graceful-fs: 4.2.10
+      micromatch: 4.0.8
+      node-gyp-build: 4.8.4
+      resolve-from: 5.0.0
+    transitivePeerDependencies:
+      - encoding
+      - rollup
+      - supports-color
+    dev: false
+
   /abbrev@1.1.1:
     resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
-    dev: true
 
   /abort-controller@3.0.0:
     resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
@@ -2074,6 +2148,14 @@ packages:
       negotiator: 0.6.3
     dev: false
 
+  /acorn-import-attributes@1.9.5(acorn@8.14.0):
+    resolution: {integrity: sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==}
+    peerDependencies:
+      acorn: ^8
+    dependencies:
+      acorn: 8.14.0
+    dev: false
+
   /acorn-node@1.8.2:
     resolution: {integrity: sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A==}
     dependencies:
@@ -2091,6 +2173,13 @@ packages:
     resolution: {integrity: sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==}
     engines: {node: '>=0.4.0'}
 
+  /acorn-walk@8.3.4:
+    resolution: {integrity: sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==}
+    engines: {node: '>=0.4.0'}
+    dependencies:
+      acorn: 8.14.0
+    dev: false
+
   /acorn@7.4.1:
     resolution: {integrity: sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==}
     engines: {node: '>=0.4.0'}
@@ -2102,12 +2191,27 @@ packages:
     engines: {node: '>=0.4.0'}
     hasBin: true
 
+  /acorn@8.14.0:
+    resolution: {integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==}
+    engines: {node: '>=0.4.0'}
+    hasBin: true
+    dev: false
+
   /acorn@8.8.0:
     resolution: {integrity: sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==}
     engines: {node: '>=0.4.0'}
     hasBin: true
     dev: false
 
+  /agent-base@6.0.2:
+    resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
+    engines: {node: '>= 6.0.0'}
+    dependencies:
+      debug: 4.3.7
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /agent-base@7.1.1:
     resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==}
     engines: {node: '>= 14'}
@@ -2186,6 +2290,19 @@ packages:
       normalize-path: 3.0.0
       picomatch: 2.3.1
 
+  /aproba@2.0.0:
+    resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
+    dev: false
+
+  /are-we-there-yet@2.0.0:
+    resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==}
+    engines: {node: '>=10'}
+    deprecated: This package is no longer supported.
+    dependencies:
+      delegates: 1.0.0
+      readable-stream: 3.6.2
+    dev: false
+
   /arg@4.1.3:
     resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
 
@@ -2248,6 +2365,10 @@ packages:
       tslib: 2.4.0
     dev: false
 
+  /async-sema@3.1.1:
+    resolution: {integrity: sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==}
+    dev: false
+
   /asynckit@0.4.0:
     resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
     dev: true
@@ -2367,6 +2488,62 @@ packages:
     transitivePeerDependencies:
       - supports-color
 
+  /ava@6.2.0:
+    resolution: {integrity: sha512-+GZk5PbyepjiO/68hzCZCUepQOQauKfNnI7sA4JukBTg97jD7E+tDKEA7OhGOGr6EorNNMM9+jqvgHVOTOzG4w==}
+    engines: {node: ^18.18 || ^20.8 || ^22 || >=23}
+    hasBin: true
+    peerDependencies:
+      '@ava/typescript': '*'
+    peerDependenciesMeta:
+      '@ava/typescript':
+        optional: true
+    dependencies:
+      '@vercel/nft': 0.27.7
+      acorn: 8.14.0
+      acorn-walk: 8.3.4
+      ansi-styles: 6.2.1
+      arrgv: 1.0.2
+      arrify: 3.0.0
+      callsites: 4.2.0
+      cbor: 9.0.2
+      chalk: 5.3.0
+      chunkd: 2.0.1
+      ci-info: 4.1.0
+      ci-parallel-vars: 1.0.1
+      cli-truncate: 4.0.0
+      code-excerpt: 4.0.0
+      common-path-prefix: 3.0.0
+      concordance: 5.0.4
+      currently-unhandled: 0.4.1
+      debug: 4.3.7
+      emittery: 1.0.3
+      figures: 6.1.0
+      globby: 14.0.2
+      ignore-by-default: 2.1.0
+      indent-string: 5.0.0
+      is-plain-object: 5.0.0
+      is-promise: 4.0.0
+      matcher: 5.0.0
+      memoize: 10.0.0
+      ms: 2.1.3
+      p-map: 7.0.2
+      package-config: 5.0.0
+      picomatch: 4.0.2
+      plur: 5.1.0
+      pretty-ms: 9.2.0
+      resolve-cwd: 3.0.0
+      stack-utils: 2.0.6
+      strip-ansi: 7.1.0
+      supertap: 3.0.1
+      temp-dir: 3.0.0
+      write-file-atomic: 6.0.0
+      yargs: 17.7.2
+    transitivePeerDependencies:
+      - encoding
+      - rollup
+      - supports-color
+    dev: false
+
   /awilix@10.0.2:
     resolution: {integrity: sha512-hFatb7eZFdtiWjjmGRSm/K/uxZpmcBlM+YoeMB3VpOPXk3xa6+7zctg3LRbUzoimom5bwGrePF0jXReO6b4zNQ==}
     engines: {node: '>=14.0.0'}
@@ -2407,6 +2584,12 @@ packages:
     resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==}
     engines: {node: '>=8'}
 
+  /bindings@1.5.0:
+    resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
+    dependencies:
+      file-uri-to-path: 1.0.0
+    dev: false
+
   /bl@4.1.0:
     resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
     dependencies:
@@ -2552,6 +2735,11 @@ packages:
     resolution: {integrity: sha512-y3jRROutgpKdz5vzEhWM34TidDU8vkJppF8dszITeb1PQmSqV3DTxyV8G/lyO/DNvtE1YTedehmw9MPZsCBHxQ==}
     engines: {node: '>=12.20'}
 
+  /callsites@4.2.0:
+    resolution: {integrity: sha512-kfzR4zzQtAE9PC7CzZsjl3aBNbXWuXiSeOCdLcPpBfGW8YuCqQHcRPFDbr/BPVmd3EEPVpuFzLyuT/cUhPr4OQ==}
+    engines: {node: '>=12.20'}
+    dev: false
+
   /camel-case@4.1.2:
     resolution: {integrity: sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==}
     dependencies:
@@ -2584,6 +2772,13 @@ packages:
     dependencies:
       nofilter: 3.1.0
 
+  /cbor@9.0.2:
+    resolution: {integrity: sha512-JPypkxsB10s9QOWwa6zwPzqE1Md3vqpPc+cai4sAecuCsRyAtAl/pMyhPlMbT/xtPnm2dznJZYRLui57qiRhaQ==}
+    engines: {node: '>=16'}
+    dependencies:
+      nofilter: 3.1.0
+    dev: false
+
   /chalk@2.4.2:
     resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==}
     engines: {node: '>=4'}
@@ -2656,7 +2851,6 @@ packages:
   /chownr@2.0.0:
     resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==}
     engines: {node: '>=10'}
-    dev: true
 
   /chunkd@2.0.1:
     resolution: {integrity: sha512-7d58XsFmOq0j6el67Ug9mHf9ELUXsQXYJBkyxhH/k+6Ke0qXRnv0kbemx+Twc6fRJ07C49lcbdgm9FL1Ei/6SQ==}
@@ -2665,6 +2859,11 @@ packages:
     resolution: {integrity: sha512-eXTggHWSooYhq49F2opQhuHWgzucfF2YgODK4e1566GQs5BIfP30B0oenwBJHfWxAs2fyPB1s7Mg949zLf61Yw==}
     engines: {node: '>=8'}
 
+  /ci-info@4.1.0:
+    resolution: {integrity: sha512-HutrvTNsF48wnxkzERIXOe5/mlcfFcbfCmwcg6CJnizbSue78AbDt+1cgl26zwn61WFxhcPykPfZrbqjGmBb4A==}
+    engines: {node: '>=8'}
+    dev: false
+
   /ci-parallel-vars@1.0.1:
     resolution: {integrity: sha512-uvzpYrpmidaoxvIQHM+rKSrigjOe9feHYbw4uOI2gdfe1C3xIlxO+kVXq83WQWNniTf8bAxVpy+cQeFQsMERKg==}
 
@@ -2707,6 +2906,14 @@ packages:
       slice-ansi: 5.0.0
       string-width: 5.1.2
 
+  /cli-truncate@4.0.0:
+    resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==}
+    engines: {node: '>=18'}
+    dependencies:
+      slice-ansi: 5.0.0
+      string-width: 7.2.0
+    dev: false
+
   /cli-width@4.0.0:
     resolution: {integrity: sha512-ZksGS2xpa/bYkNzN3BAw1wEjsLV/ZKOf/CCrJ/QOBsxx6fOARIkwTutxp1XIOIohi6HKmOFjMoK/XaqDVUpEEw==}
     engines: {node: '>= 12'}
@@ -2769,6 +2976,11 @@ packages:
   /color-name@1.1.4:
     resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
 
+  /color-support@1.1.3:
+    resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==}
+    hasBin: true
+    dev: false
+
   /colors@1.4.0:
     resolution: {integrity: sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==}
     engines: {node: '>=0.1.90'}
@@ -2805,6 +3017,10 @@ packages:
       semver: 7.5.4
       well-known-symbols: 2.0.0
 
+  /console-control-strings@1.1.0:
+    resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==}
+    dev: false
+
   /content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -2992,6 +3208,18 @@ packages:
       ms: 2.1.2
     dev: true
 
+  /debug@4.3.7:
+    resolution: {integrity: sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==}
+    engines: {node: '>=6.0'}
+    peerDependencies:
+      supports-color: '*'
+    peerDependenciesMeta:
+      supports-color:
+        optional: true
+    dependencies:
+      ms: 2.1.3
+    dev: false
+
   /decamelize-keys@1.1.0:
     resolution: {integrity: sha512-ocLWuYzRPoS9bfiSdDd3cxvrzovVMZnRDVEzAs+hWIVXGDbHxWMECij2OBuyB/An0FFW/nLuq6Kv1i/YC5Qfzg==}
     engines: {node: '>=0.10.0'}
@@ -3079,6 +3307,11 @@ packages:
     engines: {node: '>=8'}
     dev: true
 
+  /detect-libc@2.0.3:
+    resolution: {integrity: sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==}
+    engines: {node: '>=8'}
+    dev: false
+
   /detective@5.2.1:
     resolution: {integrity: sha512-v9XE1zRnz1wRtgurGu0Bs8uHKFSTdteYZNbIPFVhUZ39L/S79ppMpdmVOZAnoz1jfEFodc48n6MX483Xo3t1yw==}
     engines: {node: '>=0.8.0'}
@@ -3161,6 +3394,15 @@ packages:
     resolution: {integrity: sha512-2ID6FdrMD9KDLldGesP6317G78K7km/kMcwItRtVFva7I/cSEOIaLpewaUb+YLXVwdAp3Ctfxh/V5zIl1sj7dQ==}
     engines: {node: '>=14.16'}
 
+  /emittery@1.0.3:
+    resolution: {integrity: sha512-tJdCJitoy2lrC2ldJcqN4vkqJ00lT+tOWNT1hBJjO/3FDMJa5TTIiYGCKGkn/WfCyOzUMObeohbVTj00fhiLiA==}
+    engines: {node: '>=14.16'}
+    dev: false
+
+  /emoji-regex@10.4.0:
+    resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==}
+    dev: false
+
   /emoji-regex@8.0.0:
     resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==}
 
@@ -3806,6 +4048,10 @@ packages:
     engines: {node: '>=4.0'}
     dev: true
 
+  /estree-walker@2.0.2:
+    resolution: {integrity: sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==}
+    dev: false
+
   /esutils@2.0.3:
     resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==}
     engines: {node: '>=0.10.0'}
@@ -3895,7 +4141,6 @@ packages:
       glob-parent: 5.1.2
       merge2: 1.4.1
       micromatch: 4.0.8
-    dev: true
 
   /fast-json-patch@3.1.1:
     resolution: {integrity: sha512-vf6IHUX2SBcA+5/+4883dsIjpBTqmfBjmYiWK1savxQmFk4JfBMLa7ynTYOs1Rolp/T1betJxHiGD3g1Mn8lUQ==}
@@ -3931,6 +4176,17 @@ packages:
       escape-string-regexp: 5.0.0
       is-unicode-supported: 1.3.0
 
+  /figures@6.1.0:
+    resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==}
+    engines: {node: '>=18'}
+    dependencies:
+      is-unicode-supported: 2.1.0
+    dev: false
+
+  /file-uri-to-path@1.0.0:
+    resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
+    dev: false
+
   /fill-range@7.1.1:
     resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
     engines: {node: '>=8'}
@@ -3942,6 +4198,11 @@ packages:
     engines: {node: '>=14.16'}
     dev: true
 
+  /find-up-simple@1.0.0:
+    resolution: {integrity: sha512-q7Us7kcjj2VMePAa02hDAF6d+MzsdsAWEwYyOpwUtlerRBkOEPBCRZrAV4XfcSN8fHAgaD0hP7miwoay6DCprw==}
+    engines: {node: '>=18'}
+    dev: false
+
   /find-up@4.1.0:
     resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
     engines: {node: '>=8'}
@@ -4036,7 +4297,6 @@ packages:
     engines: {node: '>= 8'}
     dependencies:
       minipass: 3.3.6
-    dev: true
 
   /fs-minipass@3.0.3:
     resolution: {integrity: sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==}
@@ -4072,10 +4332,31 @@ packages:
     resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
     dev: true
 
+  /gauge@3.0.2:
+    resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==}
+    engines: {node: '>=10'}
+    deprecated: This package is no longer supported.
+    dependencies:
+      aproba: 2.0.0
+      color-support: 1.1.3
+      console-control-strings: 1.1.0
+      has-unicode: 2.0.1
+      object-assign: 4.1.1
+      signal-exit: 3.0.7
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+      wide-align: 1.1.5
+    dev: false
+
   /get-caller-file@2.0.5:
     resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==}
     engines: {node: 6.* || 8.* || >= 10.*}
 
+  /get-east-asian-width@1.3.0:
+    resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==}
+    engines: {node: '>=18'}
+    dev: false
+
   /get-intrinsic@1.1.3:
     resolution: {integrity: sha512-QJVz1Tj7MS099PevUG5jvnt9tSkXN8K14dxQlikJuPt4uD9hHAHjLyLBiLR5zELelBdD9QNRAXZzsJx0WaDL9A==}
     dependencies:
@@ -4164,9 +4445,20 @@ packages:
       merge2: 1.4.1
       slash: 4.0.0
 
+  /globby@14.0.2:
+    resolution: {integrity: sha512-s3Fq41ZVh7vbbe2PN3nrW7yC7U7MFVc5c98/iTl9c2GawNMKx/J648KQRW6WKkuU8GIbbh2IXfIRQjOZnXcTnw==}
+    engines: {node: '>=18'}
+    dependencies:
+      '@sindresorhus/merge-streams': 2.3.0
+      fast-glob: 3.3.2
+      ignore: 5.2.4
+      path-type: 5.0.0
+      slash: 5.1.0
+      unicorn-magic: 0.1.0
+    dev: false
+
   /graceful-fs@4.2.10:
     resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==}
-    dev: true
 
   /grapheme-splitter@1.0.4:
     resolution: {integrity: sha512-bzh50DW9kTPM00T8y4o8vQg89Di9oLJVLW/KaOGIXJWP/iqCN6WKYkbNOF04vFLJhwcpYUh9ydh/+5vpOqV4YQ==}
@@ -4217,6 +4509,10 @@ packages:
     dependencies:
       has-symbols: 1.0.3
 
+  /has-unicode@2.0.1:
+    resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==}
+    dev: false
+
   /has@1.0.3:
     resolution: {integrity: sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==}
     engines: {node: '>= 0.4.0'}
@@ -4294,6 +4590,16 @@ packages:
   /http-status-codes@2.3.0:
     resolution: {integrity: sha512-RJ8XvFvpPM/Dmc5SV+dC4y5PCeOhT3x1Hq0NU3rjGeg5a/CqlhZ7uudknPwZFz4aeAXDcbAyaeP7GAo9lvngtA==}
 
+  /https-proxy-agent@5.0.1:
+    resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
+    engines: {node: '>= 6'}
+    dependencies:
+      agent-base: 6.0.2
+      debug: 4.3.7
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
   /https-proxy-agent@7.0.5:
     resolution: {integrity: sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==}
     engines: {node: '>= 14'}
@@ -4598,6 +4904,11 @@ packages:
     resolution: {integrity: sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==}
     engines: {node: '>=12'}
 
+  /is-unicode-supported@2.1.0:
+    resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==}
+    engines: {node: '>=18'}
+    dev: false
+
   /is-utf8@0.2.1:
     resolution: {integrity: sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q==}
 
@@ -4916,6 +5227,13 @@ packages:
     dependencies:
       yallist: 4.0.0
 
+  /make-dir@3.1.0:
+    resolution: {integrity: sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==}
+    engines: {node: '>=8'}
+    dependencies:
+      semver: 6.3.1
+    dev: false
+
   /make-error@1.3.6:
     resolution: {integrity: sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==}
 
@@ -4979,6 +5297,13 @@ packages:
       map-age-cleaner: 0.1.3
       mimic-fn: 4.0.0
 
+  /memoize@10.0.0:
+    resolution: {integrity: sha512-H6cBLgsi6vMWOcCpvVCdFFnl3kerEXbrYh9q+lY6VXvQSmM6CkmV08VOwT+WE2tzIEqRPFfAq3fm4v/UIW6mSA==}
+    engines: {node: '>=18'}
+    dependencies:
+      mimic-function: 5.0.1
+    dev: false
+
   /meow@6.1.1:
     resolution: {integrity: sha512-3YffViIt2QWgTy6Pale5QpopX/IvU3LPL03jOTqp6pGj3VjesdO/U8CuHMKpnQr4shCNCM5fd5XFFvIIl6JBHg==}
     engines: {node: '>=8'}
@@ -5034,6 +5359,11 @@ packages:
     resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==}
     engines: {node: '>=12'}
 
+  /mimic-function@5.0.1:
+    resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==}
+    engines: {node: '>=18'}
+    dev: false
+
   /min-indent@1.0.1:
     resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==}
     engines: {node: '>=4'}
@@ -5113,12 +5443,10 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       yallist: 4.0.0
-    dev: true
 
   /minipass@5.0.0:
     resolution: {integrity: sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==}
     engines: {node: '>=8'}
-    dev: true
 
   /minipass@7.1.2:
     resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
@@ -5131,7 +5459,6 @@ packages:
     dependencies:
       minipass: 3.3.6
       yallist: 4.0.0
-    dev: true
 
   /mixme@0.5.4:
     resolution: {integrity: sha512-3KYa4m4Vlqx98GPdOHghxSdNtTvcP8E0kkaJ5Dlh+h2DRzF7zpuVVcA8B0QpKd11YJeP9QQ7ASkKzOeu195Wzw==}
@@ -5142,7 +5469,6 @@ packages:
     resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==}
     engines: {node: '>=10'}
     hasBin: true
-    dev: true
 
   /mock-fs@5.1.4:
     resolution: {integrity: sha512-sudhLjCjX37qWIcAlIv1OnAxB2wI4EmXByVuUjILh1rKGNGpGU8GNnzw+EAbrhdpBe0TL/KONbK1y3RXZk8SxQ==}
@@ -5196,6 +5522,11 @@ packages:
       whatwg-url: 5.0.0
     dev: false
 
+  /node-gyp-build@4.8.4:
+    resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}
+    hasBin: true
+    dev: false
+
   /nodemon@3.0.1:
     resolution: {integrity: sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==}
     engines: {node: '>=10'}
@@ -5224,6 +5555,14 @@ packages:
       abbrev: 1.1.1
     dev: true
 
+  /nopt@5.0.0:
+    resolution: {integrity: sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==}
+    engines: {node: '>=6'}
+    hasBin: true
+    dependencies:
+      abbrev: 1.1.1
+    dev: false
+
   /normalize-package-data@2.5.0:
     resolution: {integrity: sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==}
     dependencies:
@@ -5270,6 +5609,16 @@ packages:
       path-key: 3.1.1
     dev: true
 
+  /npmlog@5.0.1:
+    resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==}
+    deprecated: This package is no longer supported.
+    dependencies:
+      are-we-there-yet: 2.0.0
+      console-control-strings: 1.1.0
+      gauge: 3.0.2
+      set-blocking: 2.0.0
+    dev: false
+
   /nth-check@2.1.1:
     resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
     dependencies:
@@ -5279,7 +5628,6 @@ packages:
   /object-assign@4.1.1:
     resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==}
     engines: {node: '>=0.10.0'}
-    dev: true
 
   /object-hash@3.0.0:
     resolution: {integrity: sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw==}
@@ -5446,6 +5794,11 @@ packages:
     dependencies:
       aggregate-error: 4.0.1
 
+  /p-map@7.0.2:
+    resolution: {integrity: sha512-z4cYYMMdKHzw4O5UkWJImbZynVIo0lSGTXc7bzB1e/rrDqkgGUNysK/o4bTr+0+xKvvLoTyGqYC4Fgljy9qe1Q==}
+    engines: {node: '>=18'}
+    dev: false
+
   /p-queue@6.6.2:
     resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==}
     engines: {node: '>=8'}
@@ -5478,6 +5831,14 @@ packages:
     engines: {node: '>=6'}
     dev: true
 
+  /package-config@5.0.0:
+    resolution: {integrity: sha512-GYTTew2slBcYdvRHqjhwaaydVMvn/qrGC323+nKclYioNSLTDUM/lGgtGTgyHVtYcozb+XkE8CNhwcraOmZ9Mg==}
+    engines: {node: '>=18'}
+    dependencies:
+      find-up-simple: 1.0.0
+      load-json-file: 7.0.1
+    dev: false
+
   /package-json-from-dist@1.0.0:
     resolution: {integrity: sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==}
     dev: true
@@ -5507,6 +5868,11 @@ packages:
     resolution: {integrity: sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==}
     engines: {node: '>=12'}
 
+  /parse-ms@4.0.0:
+    resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==}
+    engines: {node: '>=18'}
+    dev: false
+
   /parse5-htmlparser2-tree-adapter@7.0.0:
     resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==}
     dependencies:
@@ -5580,6 +5946,11 @@ packages:
     resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}
     engines: {node: '>=8'}
 
+  /path-type@5.0.0:
+    resolution: {integrity: sha512-5HviZNaZcfqP95rwpv+1HDgUamezbqdSYTyzjTvwtJSnIH+3vnbmWsItli8OFEndS984VT55M3jduxZbX351gg==}
+    engines: {node: '>=12'}
+    dev: false
+
   /peek-stream@1.1.3:
     resolution: {integrity: sha512-FhJ+YbOSBb9/rIl2ZeE/QHEsWn7PqNYt8ARAY3kIgNGOk13g9FGyIY6JIl/xB/3TFRVoTv5as0l11weORrTekA==}
     dependencies:
@@ -5608,6 +5979,11 @@ packages:
     resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==}
     engines: {node: '>=8.6'}
 
+  /picomatch@4.0.2:
+    resolution: {integrity: sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==}
+    engines: {node: '>=12'}
+    dev: false
+
   /pify@2.3.0:
     resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==}
     engines: {node: '>=0.10.0'}
@@ -5808,6 +6184,13 @@ packages:
     dependencies:
       parse-ms: 3.0.0
 
+  /pretty-ms@9.2.0:
+    resolution: {integrity: sha512-4yf0QO/sllf/1zbZWYnvWw3NxCQwLXKzIj0G849LSufP15BXKM0rbD2Z3wVnkMfjdn/CB0Dpp444gYAACdsplg==}
+    engines: {node: '>=18'}
+    dependencies:
+      parse-ms: 4.0.0
+    dev: false
+
   /proc-log@4.2.0:
     resolution: {integrity: sha512-g8+OnU/L2v+wyiVK+D5fA34J7EH8jZ8DDlvwhRCMxmMj7UCBvxiO1mGeN+36JXIKF4zevU4kRBd8lVgG9vLelA==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -5976,7 +6359,6 @@ packages:
       inherits: 2.0.4
       string_decoder: 1.3.0
       util-deprecate: 1.0.2
-    dev: true
 
   /readable-stream@4.2.0:
     resolution: {integrity: sha512-gJrBHsaI3lgBoGMW/jHZsQ/o/TIWiu5ENCJG1BB7fuCKzpFM8GaS2UoBVt9NO+oI+3FcrBNbUkl3ilDe09aY4A==}
@@ -6143,6 +6525,11 @@ packages:
     hasBin: true
     dev: true
 
+  /semver@6.3.1:
+    resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
+    hasBin: true
+    dev: false
+
   /semver@7.5.4:
     resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==}
     engines: {node: '>=10'}
@@ -6154,7 +6541,6 @@ packages:
     resolution: {integrity: sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==}
     engines: {node: '>=10'}
     hasBin: true
-    dev: true
 
   /serialize-error@7.0.1:
     resolution: {integrity: sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==}
@@ -6164,7 +6550,6 @@ packages:
 
   /set-blocking@2.0.0:
     resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==}
-    dev: true
 
   /setprototypeof@1.2.0:
     resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==}
@@ -6203,7 +6588,6 @@ packages:
 
   /signal-exit@3.0.7:
     resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==}
-    dev: true
 
   /signal-exit@4.0.2:
     resolution: {integrity: sha512-MY2/qGx4enyjprQnFaZsHib3Yadh3IXyV2C321GY0pjGfVBu4un0uDJkwgdxqO+Rdx8JMT8IfJIRwbYVz3Ob3Q==}
@@ -6212,7 +6596,6 @@ packages:
   /signal-exit@4.1.0:
     resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
     engines: {node: '>=14'}
-    dev: true
 
   /simple-update-notifier@2.0.0:
     resolution: {integrity: sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==}
@@ -6230,6 +6613,11 @@ packages:
     resolution: {integrity: sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==}
     engines: {node: '>=12'}
 
+  /slash@5.1.0:
+    resolution: {integrity: sha512-ZA6oR3T/pEyuqwMgAKT0/hAv8oAXckzbkmR0UkUosQ+Mc4RxGoJkRmwHgHufaenlyAgE1Mxgpdcrf75y6XcnDg==}
+    engines: {node: '>=14.16'}
+    dev: false
+
   /slice-ansi@5.0.0:
     resolution: {integrity: sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==}
     engines: {node: '>=12'}
@@ -6393,6 +6781,15 @@ packages:
       emoji-regex: 9.2.2
       strip-ansi: 7.1.0
 
+  /string-width@7.2.0:
+    resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==}
+    engines: {node: '>=18'}
+    dependencies:
+      emoji-regex: 10.4.0
+      get-east-asian-width: 1.3.0
+      strip-ansi: 7.1.0
+    dev: false
+
   /string.prototype.trimend@1.0.5:
     resolution: {integrity: sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog==}
     dependencies:
@@ -6419,7 +6816,6 @@ packages:
     resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==}
     dependencies:
       safe-buffer: 5.2.1
-    dev: true
 
   /strip-ansi@6.0.1:
     resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==}
@@ -6547,7 +6943,6 @@ packages:
       minizlib: 2.1.2
       mkdirp: 1.0.4
       yallist: 4.0.0
-    dev: true
 
   /temp-dir@3.0.0:
     resolution: {integrity: sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==}
@@ -7043,6 +7438,16 @@ packages:
     dependencies:
       '@fastify/busboy': 2.1.1
 
+  /undici@7.1.0:
+    resolution: {integrity: sha512-3+mdX2R31khuLCm2mKExSlMdJsfol7bJkIMH80tdXA74W34rT1jKemUTlYR7WY3TqsV4wfOgpatWmmB2Jl1+5g==}
+    engines: {node: '>=20.18.1'}
+    dev: false
+
+  /unicorn-magic@0.1.0:
+    resolution: {integrity: sha512-lRfVq8fE8gz6QMBuDM6a+LO3IAzTi05H6gCVaUpir2E1Rwpo4ZUog45KpNXKC/Mn3Yb9UDuHumeFTo9iV/D9FQ==}
+    engines: {node: '>=18'}
+    dev: false
+
   /unique-filename@3.0.0:
     resolution: {integrity: sha512-afXhuC55wkAmZ0P18QsVE6kp8JaxrEokN2HGIoIVv2ijHQd419H0+6EigAFcIzXeMIkcIkNBpB3L/DXB3cTS/g==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -7074,7 +7479,6 @@ packages:
 
   /util-deprecate@1.0.2:
     resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
-    dev: true
 
   /v8-compile-cache-lib@3.0.1:
     resolution: {integrity: sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==}
@@ -7166,6 +7570,12 @@ packages:
       isexe: 2.0.0
     dev: true
 
+  /wide-align@1.1.5:
+    resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==}
+    dependencies:
+      string-width: 4.2.3
+    dev: false
+
   /word-wrap@1.2.5:
     resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==}
     engines: {node: '>=0.10.0'}
@@ -7210,6 +7620,14 @@ packages:
       imurmurhash: 0.1.4
       signal-exit: 4.0.2
 
+  /write-file-atomic@6.0.0:
+    resolution: {integrity: sha512-GmqrO8WJ1NuzJ2DrziEI2o57jKAVIQNf8a18W3nCYU3H7PNWqCCVTeH6/NQE93CIllIgQS98rrmVkYgTX9fFJQ==}
+    engines: {node: ^18.17.0 || >=20.5.0}
+    dependencies:
+      imurmurhash: 0.1.4
+      signal-exit: 4.1.0
+    dev: false
+
   /ws@8.18.0:
     resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==}
     engines: {node: '>=10.0.0'}

From 778b3d1d825d7f96a957db8d99b2480e856d8bc4 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Thu, 5 Dec 2024 17:03:55 +0000
Subject: [PATCH 02/23] collections: fix output default

---
 packages/cli/src/collections/command.ts | 15 ++++++++--
 packages/cli/src/collections/handler.ts | 37 +++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 79456d1a5..b2bf1e04b 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -105,12 +105,23 @@ const getOptions = [
     default: 'info',
   }),
   o.logJson,
-  o.outputPath,
+  {
+    ...o.outputPath,
+    // disable default output path behaviour
+    ensure: () => {},
+  },
 ];
 
 export const get = {
-  command: 'get [name] [key]',
+  command: 'get name key',
   describe: 'Get values from a collection',
   handler: ensure('collections-get', getOptions),
   builder: (yargs) => build(getOptions, yargs),
 } as yargs.CommandModule<{}>;
+
+export const set = {
+  command: 'set name [key] path',
+  describe: 'Uploads values to a collection',
+  handler: ensure('collections-get', getOptions),
+  builder: (yargs) => build(getOptions, yargs),
+} as yargs.CommandModule<{}>;
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 2075eef90..9b4d0473f 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -38,6 +38,43 @@ export const get = async (options: CollectionsOptions, logger: Logger) => {
   }
 };
 
+export const set = async (options: CollectionsOptions, logger: Logger) => {
+  logger.info(
+    `Fetching "${options.key}" from collection "${options.collectionName}"`
+  );
+
+  // get the input data
+
+  const result = await request(
+    'GET',
+    {
+      lightning: options.lightning,
+      token: options.token,
+      pageSize: options.pageSize,
+      limit: options.limit,
+      key: options.key,
+      collectionName: options.collectionName,
+    },
+    logger
+  );
+
+  logger.success(`Fetched ${result.count} items!`);
+
+  if (options.outputPath) {
+    const content = JSON.stringify(
+      result,
+      null,
+      options.pretty ? 2 : undefined
+    );
+    await writeFile(options.outputPath!, content);
+    logger.always(`Wrote items to ${options.outputPath}`);
+  } else {
+    // use print because it won't stringify
+    logger.print(result.items);
+  }
+};
+
 export default {
   get,
+  set,
 };

From 71cd2a6355efcd3d912e08d022c9b695f15f9102 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 6 Dec 2024 12:16:54 +0000
Subject: [PATCH 03/23] cli: add collections-set support

---
 packages/cli/src/collections/command.ts   | 67 +++++++++++++++++---
 packages/cli/src/collections/handler.ts   | 74 +++++++++++++++--------
 packages/cli/src/collections/request.ts   | 52 +++++++++++-----
 packages/cli/src/commands.ts              |  4 +-
 packages/cli/src/util/command-builders.ts |  1 +
 5 files changed, 147 insertions(+), 51 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index b2bf1e04b..4738d3dca 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -3,27 +3,37 @@ import * as o from '../options';
 import type { Opts } from '../options';
 import { build, ensure, override } from '../util/command-builders';
 
+// TODO output options are only relevant to get
 export type CollectionsOptions = Pick<
   Opts,
   'log' | 'logJson' | 'outputPath' | 'outputStdout'
 > & {
   lightning?: string;
   token?: string;
-  pageSize?: number;
-  limit?: number;
   key: string;
   collectionName: string;
+};
+
+export type GetOptions = CollectionsOptions & {
+  pageSize?: number;
+  limit?: number;
   pretty?: boolean;
 };
 
+export type SetOptions = CollectionsOptions & {
+  items?: string;
+  value?: string;
+};
+
 const desc = `Call out to the Collections API on Lightning`;
 
 export default {
-  command: 'collections [subcommand]',
+  command: 'collections <subcommand>',
   describe: desc,
   builder: (yargs) =>
     yargs
       .command(get)
+      .command(set)
       .example(
         'collections get my-collection 2024* -O',
         'Get all keys from my-collection starting with the string "2024" and log the results to stdout'
@@ -100,7 +110,6 @@ const getOptions = [
   limit,
   pretty,
 
-  o.stateStdin,
   override(o.log, {
     default: 'info',
   }),
@@ -113,15 +122,55 @@ const getOptions = [
 ];
 
 export const get = {
-  command: 'get name key',
+  command: 'get <name> <key>',
   describe: 'Get values from a collection',
   handler: ensure('collections-get', getOptions),
   builder: (yargs) => build(getOptions, yargs),
 } as yargs.CommandModule<{}>;
 
+const value = {
+  name: 'value',
+  yargs: {
+    description: 'Path to the value to upsert',
+  },
+};
+
+const items = {
+  name: 'items',
+  yargs: {
+    description:
+      'Path to a batch of items to upsert. Must contain a JSON object where each key is an item key, and each value is an uploaded value',
+  },
+};
+
+const setOptions = [
+  collectionName,
+  override(key, {
+    demand: false,
+  }),
+  token,
+  lightningUrl,
+  value,
+  items,
+
+  override(o.log, {
+    default: 'info',
+  }),
+  o.logJson,
+];
+
 export const set = {
-  command: 'set name [key] path',
-  describe: 'Uploads values to a collection',
-  handler: ensure('collections-get', getOptions),
-  builder: (yargs) => build(getOptions, yargs),
+  command: 'set <name> [key] [value] [--items]',
+  describe: 'Uploads values to a collection. Must set key & value OR --items.',
+  handler: ensure('collections-set', setOptions),
+  builder: (yargs) =>
+    build(setOptions, yargs)
+      .example(
+        'collections set my-collection cities-mapping ./citymap.json',
+        'Upload the data in ./citymap.json to the cities-mapping key'
+      )
+      .example(
+        'collections set my-collection --items ./items.json',
+        'Upsert the object in ./items.json as a batch of items (key/value pairs)'
+      ),
 } as yargs.CommandModule<{}>;
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 9b4d0473f..4b9502572 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -1,14 +1,19 @@
-import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { readFile, writeFile } from 'node:fs/promises';
 
 import { Logger } from '../util/logger';
-import { CollectionsOptions } from './command';
 import request from './request';
 
-export const get = async (options: CollectionsOptions, logger: Logger) => {
+import type { GetOptions, SetOptions } from './command';
+
+export const get = async (options: GetOptions, logger: Logger) => {
   logger.info(
     `Fetching "${options.key}" from collection "${options.collectionName}"`
   );
 
+  // TODO: log the output format
+  // Something like: downloading single values item vs downloading multiple key/value pairs
+
   const result = await request(
     'GET',
     {
@@ -22,6 +27,10 @@ export const get = async (options: CollectionsOptions, logger: Logger) => {
     logger
   );
 
+  result.count = Object.keys(result.items).length;
+
+  // TODO if fetching a single ite, (no pattern) return it verbatim
+
   logger.success(`Fetched ${result.count} items!`);
 
   if (options.outputPath) {
@@ -38,40 +47,55 @@ export const get = async (options: CollectionsOptions, logger: Logger) => {
   }
 };
 
-export const set = async (options: CollectionsOptions, logger: Logger) => {
-  logger.info(
-    `Fetching "${options.key}" from collection "${options.collectionName}"`
-  );
+export const set = async (options: SetOptions, logger: Logger) => {
+  logger.info(`Upserting items to collection "${options.collectionName}"`);
 
-  // get the input data
+  // Array of key/value pairs to upsert
+  const items = [];
 
+  // set multiple items
+  if (options.items) {
+    const resolvedPath = path.resolve(options.items);
+    logger.debug('Loading items from ', resolvedPath);
+    const data = await readFile(resolvedPath, 'utf8');
+    const obj = JSON.parse(data);
+
+    Object.entries(obj).forEach(([key, value]) => {
+      items.push({ key, value: JSON.stringify(value) });
+    });
+
+    logger.info(`Upserting ${items.length} items`);
+  } else if (options.key && options.value) {
+    const resolvedPath = path.resolve(options.value);
+    logger.debug('Loading value from ', resolvedPath);
+    // TODO throw if key contains a *
+
+    // set a single item
+    const data = await readFile(path.resolve(options.value), 'utf8');
+    // Ensure the data is properly jsonified
+    const value = JSON.stringify(JSON.parse(data));
+
+    items.push({ key: options.key, value });
+    logger.info(`Upserting data to "${options.key}"`);
+  } else {
+    // throw for invalid arguments
+    throw new Error('INVALID_ARGUMENTS');
+  }
+  console.log(items);
+  // get the input data
   const result = await request(
-    'GET',
+    'POST',
     {
       lightning: options.lightning,
       token: options.token,
-      pageSize: options.pageSize,
-      limit: options.limit,
       key: options.key,
       collectionName: options.collectionName,
+      data: { items },
     },
     logger
   );
 
-  logger.success(`Fetched ${result.count} items!`);
-
-  if (options.outputPath) {
-    const content = JSON.stringify(
-      result,
-      null,
-      options.pretty ? 2 : undefined
-    );
-    await writeFile(options.outputPath!, content);
-    logger.always(`Wrote items to ${options.outputPath}`);
-  } else {
-    // use print because it won't stringify
-    logger.print(result.items);
-  }
+  logger.success(`Upserted ${result.upserted} items!`);
 };
 
 export default {
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 95b78381d..33d4d8db5 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -1,5 +1,6 @@
 import path from 'node:path';
 import { request } from 'undici';
+import type { Dispatcher } from 'undici';
 import { Logger } from '../util';
 
 // helper function to call out to the collections API
@@ -14,6 +15,8 @@ type Options = {
 
   includeMeta?: boolean; // TODO ignored right now
   pageSize?: number;
+
+  data?: any;
 };
 
 type Key = string;
@@ -73,13 +76,20 @@ export default async (
     limit: options.pageSize || 1000,
   };
 
-  const args = {
+  const args: Partial<Dispatcher.RequestOptions> = {
     headers,
     method,
     query,
   };
 
-  const result: ItemSet = {
+  if (options.data) {
+    args.body = JSON.stringify(options.data);
+    args.headers['content-type'] = 'application/json';
+  }
+
+  // hmm, this is the result for paging a get
+  // not quite what we need
+  let result: ItemSet = {
     count: 0, // Set the count here so that it comes up first when serialized
     items: {},
   };
@@ -95,23 +105,33 @@ export default async (
       // await handleError(response, path, state.configuration.collections_endpoint);
       logger.error('error!');
     }
-    const items: any = await response.body.json();
-    logger.debug(
-      'Received',
-      response.statusCode,
-      `- ${items.items.length} values`
-    );
-    for (const item of items.items) {
-      try {
-        result.items[item.key] = JSON.parse(item.value);
-      } catch (e) {
-        result.items[item.key] = item.value;
+    const responseData: any = await response.body.json();
+
+    if (responseData.items) {
+      // Handle a get response
+      logger.debug(
+        'Received',
+        response.statusCode,
+        `- ${responseData.items.length} values`
+      );
+      for (const item of responseData?.items) {
+        try {
+          result.items[item.key] = JSON.parse(item.value);
+        } catch (e) {
+          result.items[item.key] = item.value;
+        }
       }
+      cursor = responseData.cursor;
+    } else {
+      // handle a set response
+      logger.debug(
+        'Received',
+        response.statusCode,
+        `- ${JSON.stringify(responseData)}`
+      );
+      result = responseData;
     }
-    cursor = items.cursor;
   } while (cursor);
 
-  result.count = Object.keys(result.items).length;
-
   return result;
 };
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index f2906a65a..0c61e14d4 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -21,6 +21,7 @@ export type CommandList =
   | 'apollo'
   | 'compile'
   | 'collections-get'
+  | 'collections-set'
   | 'deploy'
   | 'docgen'
   | 'docs'
@@ -45,6 +46,7 @@ const handlers = {
   metadata,
   pull,
   ['collections-get']: collections.get,
+  ['collections-set']: collections.set,
   ['repo-clean']: clean,
   ['repo-install']: install,
   ['repo-pwd']: pwd,
@@ -90,7 +92,7 @@ const parse = async (options: Opts, log?: Logger) => {
   // TODO it would be nice to do this in the repoDir option, but
   // the logger isn't available yet
   if (
-    !/^(pull|deploy|test|version|apollo|collections-get)$/.test(
+    !/^(pull|deploy|test|version|apollo|collections-get|collections-set)$/.test(
       options.command!
     ) &&
     !options.repoDir
diff --git a/packages/cli/src/util/command-builders.ts b/packages/cli/src/util/command-builders.ts
index 3cebdb11d..980a8da2f 100644
--- a/packages/cli/src/util/command-builders.ts
+++ b/packages/cli/src/util/command-builders.ts
@@ -13,6 +13,7 @@ const expandYargs = (y: {} | (() => any)) => {
 
 // build helper to chain options
 export function build(opts: CLIOption[], yargs: yargs.Argv<any>) {
+  console.log(yargs.argv);
   return opts.reduce(
     (_y, o) => yargs.option(o.name, expandYargs(o.yargs)),
     yargs

From 2bcd441defc027a62d96f71cbe67f7d21d73d3fb Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 6 Dec 2024 12:40:16 +0000
Subject: [PATCH 04/23] cli: adjust get data

---
 packages/cli/src/collections/handler.ts   | 34 +++++++++++++----------
 packages/cli/src/util/command-builders.ts |  1 -
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 4b9502572..ec9b775b1 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -7,14 +7,18 @@ import request from './request';
 import type { GetOptions, SetOptions } from './command';
 
 export const get = async (options: GetOptions, logger: Logger) => {
-  logger.info(
-    `Fetching "${options.key}" from collection "${options.collectionName}"`
-  );
-
-  // TODO: log the output format
-  // Something like: downloading single values item vs downloading multiple key/value pairs
+  const multiMode = options.key.includes('*');
+  if (multiMode) {
+    logger.info(
+      `Fetching multiple items from collection "${options.collectionName}" with pattern ${options.key}`
+    );
+  } else {
+    logger.info(
+      `Fetching "${options.key}" from collection "${options.collectionName}"`
+    );
+  }
 
-  const result = await request(
+  let result = await request(
     'GET',
     {
       lightning: options.lightning,
@@ -27,11 +31,13 @@ export const get = async (options: GetOptions, logger: Logger) => {
     logger
   );
 
-  result.count = Object.keys(result.items).length;
-
-  // TODO if fetching a single ite, (no pattern) return it verbatim
-
-  logger.success(`Fetched ${result.count} items!`);
+  if (multiMode) {
+    result.count = Object.keys(result.items).length;
+    logger.success(`Fetched ${result.count} items!`);
+  } else {
+    result = Object.values(result.items)[0];
+    logger.success(`Fetched ${options.key}`);
+  }
 
   if (options.outputPath) {
     const content = JSON.stringify(
@@ -43,7 +49,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
     logger.always(`Wrote items to ${options.outputPath}`);
   } else {
     // use print because it won't stringify
-    logger.print(result.items);
+    logger.print(multiMode ? result.items : result);
   }
 };
 
@@ -81,7 +87,7 @@ export const set = async (options: SetOptions, logger: Logger) => {
     // throw for invalid arguments
     throw new Error('INVALID_ARGUMENTS');
   }
-  console.log(items);
+
   // get the input data
   const result = await request(
     'POST',
diff --git a/packages/cli/src/util/command-builders.ts b/packages/cli/src/util/command-builders.ts
index 980a8da2f..3cebdb11d 100644
--- a/packages/cli/src/util/command-builders.ts
+++ b/packages/cli/src/util/command-builders.ts
@@ -13,7 +13,6 @@ const expandYargs = (y: {} | (() => any)) => {
 
 // build helper to chain options
 export function build(opts: CLIOption[], yargs: yargs.Argv<any>) {
-  console.log(yargs.argv);
   return opts.reduce(
     (_y, o) => yargs.option(o.name, expandYargs(o.yargs)),
     yargs

From ce221d70e29e41e1306ad7d3bc128af701ca8a11 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 6 Dec 2024 17:09:33 +0000
Subject: [PATCH 05/23] cli: tidy up PAT access

---
 packages/cli/src/collections/handler.ts | 28 ++++++++++++++++++++++++-
 packages/cli/src/commands.ts            |  2 +-
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index ec9b775b1..1de621701 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -4,9 +4,34 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { Logger } from '../util/logger';
 import request from './request';
 
-import type { GetOptions, SetOptions } from './command';
+import type { CollectionsOptions, GetOptions, SetOptions } from './command';
+
+const ensureToken = (opts: CollectionsOptions, logger: Logger) => {
+  if (!('token' in opts)) {
+    if (process.env.OPENFN_PAT) {
+      const token = process.env.OPENFN_PAT;
+      logger.info(
+        `Using access token ending in ${token?.substring(
+          token.length - 10
+        )} from env (OPENFN_PAT)`
+      );
+      opts.token = token;
+    } else {
+      logger.error('No access token detected!');
+      logger.error(
+        'Ensure you pass a Personal Access Token (PAT) with --token $MY_TOKEN or set the OPENFN_PAT env var'
+      );
+      logger.error(
+        'You can get a PAT from OpenFn, see https://docs.openfn.org/documentation/api-tokens'
+      );
+
+      throw new Error('NO_PAT');
+    }
+  }
+};
 
 export const get = async (options: GetOptions, logger: Logger) => {
+  ensureToken(options, logger);
   const multiMode = options.key.includes('*');
   if (multiMode) {
     logger.info(
@@ -54,6 +79,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
 };
 
 export const set = async (options: SetOptions, logger: Logger) => {
+  ensureToken(options, logger);
   logger.info(`Upserting items to collection "${options.collectionName}"`);
 
   // Array of key/value pairs to upsert
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index 0c61e14d4..dbe865e28 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -87,7 +87,7 @@ const parse = async (options: Opts, log?: Logger) => {
   }
 
   // TODO Please fix this joe!
-  // Put the validation isnide the repoDir option
+  // Put the validation inside the repoDir option
 
   // TODO it would be nice to do this in the repoDir option, but
   // the logger isn't available yet

From 0755630c81d727d6cec6a171052e0321ddcc504c Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 6 Dec 2024 17:31:03 +0000
Subject: [PATCH 06/23] cli: add collections. remove

---
 packages/cli/src/collections/command.ts | 49 ++++++++++++++++++-----
 packages/cli/src/collections/handler.ts | 52 ++++++++++++++++++++++++-
 packages/cli/src/collections/request.ts |  2 +-
 packages/cli/src/commands.ts            |  2 +
 4 files changed, 94 insertions(+), 11 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 4738d3dca..bd8ea0482 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -3,21 +3,22 @@ import * as o from '../options';
 import type { Opts } from '../options';
 import { build, ensure, override } from '../util/command-builders';
 
-// TODO output options are only relevant to get
-export type CollectionsOptions = Pick<
-  Opts,
-  'log' | 'logJson' | 'outputPath' | 'outputStdout'
-> & {
+export type CollectionsOptions = Pick<Opts, 'log' | 'logJson'> & {
   lightning?: string;
   token?: string;
   key: string;
   collectionName: string;
 };
 
-export type GetOptions = CollectionsOptions & {
-  pageSize?: number;
-  limit?: number;
-  pretty?: boolean;
+export type GetOptions = CollectionsOptions &
+  Pick<Opts, 'outputPath' | 'outputStdout'> & {
+    pageSize?: number;
+    limit?: number;
+    pretty?: boolean;
+  };
+
+export type RemoveOptions = CollectionsOptions & {
+  dryRun?: boolean;
 };
 
 export type SetOptions = CollectionsOptions & {
@@ -34,6 +35,7 @@ export default {
     yargs
       .command(get)
       .command(set)
+      .command(remove)
       .example(
         'collections get my-collection 2024* -O',
         'Get all keys from my-collection starting with the string "2024" and log the results to stdout'
@@ -128,6 +130,35 @@ export const get = {
   builder: (yargs) => build(getOptions, yargs),
 } as yargs.CommandModule<{}>;
 
+const dryRun = {
+  name: 'dry-run',
+  yargs: {
+    description:
+      '[Alpha] Do not delete keys and instead return the keys that would be deleted',
+    type: 'boolean',
+  },
+};
+
+const removeOptions = [
+  collectionName,
+  key,
+  token,
+  lightningUrl,
+  dryRun,
+
+  override(o.log, {
+    default: 'info',
+  }),
+  o.logJson,
+];
+
+export const remove = {
+  command: 'remove <name> <key>',
+  describe: 'Remove values from a collection',
+  handler: ensure('collections-remove', removeOptions),
+  builder: (yargs) => build(removeOptions, yargs),
+} as yargs.CommandModule<{}>;
+
 const value = {
   name: 'value',
   yargs: {
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 1de621701..8ade607d1 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -4,7 +4,12 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { Logger } from '../util/logger';
 import request from './request';
 
-import type { CollectionsOptions, GetOptions, SetOptions } from './command';
+import type {
+  CollectionsOptions,
+  GetOptions,
+  RemoveOptions,
+  SetOptions,
+} from './command';
 
 const ensureToken = (opts: CollectionsOptions, logger: Logger) => {
   if (!('token' in opts)) {
@@ -130,7 +135,52 @@ export const set = async (options: SetOptions, logger: Logger) => {
   logger.success(`Upserted ${result.upserted} items!`);
 };
 
+export const remove = async (options: RemoveOptions, logger: Logger) => {
+  ensureToken(options, logger);
+  logger.info(
+    `Removing "${options.key}" from collection "${options.collectionName}"`
+  );
+
+  // TODO should we ALWAYS do this to report the keys that will be lost
+  // But we can't guarantee 100% accuracy if a key is inserted between queries
+  // Can we even guarantee that the query in get and delete is the same?
+  if (options.dryRun) {
+    logger.info('--dry-run passed: fetching affected items');
+    // TODO this isn't optimal at the moment, to say the least
+    // See https://github.com/OpenFn/lightning/issues/2758
+    let result = await request(
+      'GET',
+      {
+        lightning: options.lightning,
+        token: options.token,
+        key: options.key,
+        collectionName: options.collectionName,
+      },
+      logger
+    );
+    const keys = Object.keys(result.items);
+    logger.info('Keys to be removed:');
+    logger.print(keys);
+
+    logger.always('Aborting request - keys have not been removed');
+  } else {
+    let result = await request(
+      'DELETE',
+      {
+        lightning: options.lightning,
+        token: options.token,
+        key: options.key,
+        collectionName: options.collectionName,
+      },
+      logger
+    );
+
+    logger.success(`Removed ${result.deleted} items`);
+  }
+};
+
 export default {
   get,
   set,
+  remove,
 };
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 33d4d8db5..57b967a72 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -48,7 +48,7 @@ type ItemSet = {
 // TODO how should we handle cursor?
 // Lets a) support limit and b) fetch everything
 export default async (
-  method: 'GET' | 'POST',
+  method: 'GET' | 'POST' | 'DELETE',
   options: Options,
   logger: Logger
 ) => {
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index dbe865e28..1cadfd5f9 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -22,6 +22,7 @@ export type CommandList =
   | 'compile'
   | 'collections-get'
   | 'collections-set'
+  | 'collections-remove'
   | 'deploy'
   | 'docgen'
   | 'docs'
@@ -47,6 +48,7 @@ const handlers = {
   pull,
   ['collections-get']: collections.get,
   ['collections-set']: collections.set,
+  ['collections-remove']: collections.remove,
   ['repo-clean']: clean,
   ['repo-install']: install,
   ['repo-pwd']: pwd,

From 52f7d9377947faf24af1ee3188a530aecd203297 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Fri, 6 Dec 2024 18:30:12 +0000
Subject: [PATCH 07/23] cli: comment

---
 packages/cli/src/collections/command.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index bd8ea0482..893667a66 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -176,6 +176,8 @@ const items = {
 
 const setOptions = [
   collectionName,
+  // TODO in set, key does not support patterns
+  // We should document and catch this case
   override(key, {
     demand: false,
   }),

From 92255d69b6e06bf642d2ffdd6e9911c333b20f05 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sat, 7 Dec 2024 12:58:16 +0000
Subject: [PATCH 08/23] cli: clean up multi-get format

---
 packages/cli/src/collections/handler.ts |  5 ++--
 packages/cli/src/collections/request.ts | 39 ++-----------------------
 2 files changed, 5 insertions(+), 39 deletions(-)

diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 8ade607d1..9781dacb1 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -62,10 +62,9 @@ export const get = async (options: GetOptions, logger: Logger) => {
   );
 
   if (multiMode) {
-    result.count = Object.keys(result.items).length;
     logger.success(`Fetched ${result.count} items!`);
   } else {
-    result = Object.values(result.items)[0];
+    result = Object.values(result)[0];
     logger.success(`Fetched ${options.key}`);
   }
 
@@ -79,7 +78,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
     logger.always(`Wrote items to ${options.outputPath}`);
   } else {
     // use print because it won't stringify
-    logger.print(multiMode ? result.items : result);
+    logger.print(result);
   }
 };
 
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 57b967a72..f14caa193 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -19,34 +19,6 @@ type Options = {
   data?: any;
 };
 
-type Key = string;
-
-// This is what uploaded and downloaded data looks like
-// Uploads don't include count and meta, but they're harmless
-// For now, meta is discarded
-type ItemSet = {
-  items: Record<Key, any>;
-  count?: number;
-  meta?: Record<
-    Key,
-    {
-      updated: string;
-      created: string;
-    }
-  >;
-};
-
-// TODO we should try to autoparse strings into json right?
-// we can take a flag to not do that
-
-// TODO how should we return data?
-// As a key: value object?
-// what about metadata?
-// Let's add that as a second object
-// so you get: { items, metadata }
-
-// TODO how should we handle cursor?
-// Lets a) support limit and b) fetch everything
 export default async (
   method: 'GET' | 'POST' | 'DELETE',
   options: Options,
@@ -87,12 +59,7 @@ export default async (
     args.headers['content-type'] = 'application/json';
   }
 
-  // hmm, this is the result for paging a get
-  // not quite what we need
-  let result: ItemSet = {
-    count: 0, // Set the count here so that it comes up first when serialized
-    items: {},
-  };
+  let result: any = {};
 
   let cursor;
   do {
@@ -116,9 +83,9 @@ export default async (
       );
       for (const item of responseData?.items) {
         try {
-          result.items[item.key] = JSON.parse(item.value);
+          result[item.key] = JSON.parse(item.value);
         } catch (e) {
-          result.items[item.key] = item.value;
+          result[item.key] = item.value;
         }
       }
       cursor = responseData.cursor;

From d63dcf6b822ce8f0a7a77a0c9567f85b791ca36a Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sat, 7 Dec 2024 12:58:44 +0000
Subject: [PATCH 09/23] cli: start implementing collections tests

---
 packages/cli/package.json                     |   1 +
 .../cli/test/collections/collections.test.ts  |  99 ++++++++++++++++
 packages/cli/test/collections/command.test.ts | 106 ++++++++++++++++++
 pnpm-lock.yaml                                |  67 +++++++++++
 4 files changed, 273 insertions(+)
 create mode 100644 packages/cli/test/collections/collections.test.ts
 create mode 100644 packages/cli/test/collections/command.test.ts

diff --git a/packages/cli/package.json b/packages/cli/package.json
index e2bce04c3..a578c7218 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -33,6 +33,7 @@
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
   "devDependencies": {
+    "@openfn/language-collections": "^0.6.1",
     "@openfn/language-common": "2.0.0-rc3",
     "@openfn/lexicon": "workspace:^",
     "@types/mock-fs": "^4.13.1",
diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
new file mode 100644
index 000000000..1fad94761
--- /dev/null
+++ b/packages/cli/test/collections/collections.test.ts
@@ -0,0 +1,99 @@
+import test from 'ava';
+import { mockFs, resetMockFs } from '../util';
+
+import { get, set, remove } from '../../src/collections/handler';
+
+// test the collections handlers directly
+
+import { setGlobalDispatcher } from 'undici';
+import { createMockLogger } from '@openfn/logger';
+import { collections } from '@openfn/language-collections';
+import { lightning } from '@openfn/lexicon';
+import { readFile } from 'fs/promises';
+
+// Log as json to make testing easier
+const logger = createMockLogger('default', { level: 'debug', json: true });
+
+const COLLECTION = 'test-collection-a';
+
+let api: any;
+
+// Load k/v pairs into the collection
+// the id of each item is defaulted to the key
+const loadData = (items: Record<string, object>) => {
+  for (const key in items) {
+    api.upsert(
+      COLLECTION,
+      key,
+      JSON.stringify({
+        id: key,
+        ...items[key],
+      })
+    );
+  }
+};
+
+test.before(() => {
+  const client = collections.createMockServer('https://mock.openfn.org');
+  api = client.api;
+  setGlobalDispatcher(client.agent);
+});
+
+test.beforeEach(() => {
+  logger._reset();
+  api.reset();
+  api.createCollection(COLLECTION);
+  resetMockFs();
+});
+
+const createOptions = (opts = {}) => ({
+  lightning: 'https://mock.openfn.org',
+  collectionName: COLLECTION,
+  key: '*',
+  ...opts,
+});
+
+test.serial(
+  'should get all keys from a collection and print to stdout',
+  async (t) => {
+    loadData({
+      x: {},
+      y: {},
+    });
+    const options = createOptions();
+    await get(options, logger);
+
+    // The last log should print the data out
+    // (because we're logging to JSON we can easily inspect the raw JSON data)
+    const [level, log] = logger._history.at(-1);
+    t.deepEqual(log.message[0], {
+      x: { id: 'x' },
+      y: { id: 'y' },
+    });
+  }
+);
+
+test.serial(
+  'should get all keys from a collection and write to disk',
+  async (t) => {
+    mockFs({
+      '/tmp.json': '',
+    });
+    loadData({
+      x: {},
+      y: {},
+    });
+    const options = createOptions({
+      outputPath: '/tmp.json',
+    });
+    await get(options, logger);
+
+    const data = await readFile('/tmp.json');
+    const items = JSON.parse(data);
+
+    t.deepEqual(items, {
+      x: { id: 'x' },
+      y: { id: 'y' },
+    });
+  }
+);
diff --git a/packages/cli/test/collections/command.test.ts b/packages/cli/test/collections/command.test.ts
new file mode 100644
index 000000000..ac3da50f5
--- /dev/null
+++ b/packages/cli/test/collections/command.test.ts
@@ -0,0 +1,106 @@
+import test from 'ava';
+import yargs from 'yargs';
+import collections, {
+  GetOptions,
+  SetOptions,
+  RemoveOptions,
+} from '../../src/collections/command';
+
+// test option parsing
+const cmd = yargs().command(collections as any);
+
+const parse = (command: string) =>
+  cmd.parse(command) as yargs.Arguments<
+    GetOptions | SetOptions | RemoveOptions
+  >;
+
+test('all commands log to info by default', (t) => {
+  for (const cmd of ['get', 'set', 'remove']) {
+    const options = parse(`collections ${cmd} my-collection some-key`);
+    t.is(options.command, `collections-${cmd}`);
+    t.is(options.log?.default, 'info');
+  }
+});
+
+test('all commands accept a token', (t) => {
+  for (const cmd of ['get', 'set', 'remove']) {
+    const options = parse(
+      `collections ${cmd} my-collection some-key --token abc`
+    );
+    t.is(options.command, `collections-${cmd}`);
+    t.is(options.token, 'abc');
+  }
+});
+
+test('all commands accept a lighting url', (t) => {
+  for (const cmd of ['get', 'set', 'remove']) {
+    const options = parse(
+      `collections ${cmd} my-collection some-key --lightning app.openfn.org`
+    );
+    t.is(options.command, `collections-${cmd}`);
+    t.is(options.lightning, 'app.openfn.org');
+  }
+});
+
+test('get with name and key', (t) => {
+  const options = parse('collections get my-collection some-key');
+  t.is(options.command, 'collections-get');
+  t.is(options.collectionName, 'my-collection');
+  t.is(options.key, 'some-key');
+});
+
+test('get with name and key-pattern', (t) => {
+  const options = parse('collections get my-collection *');
+  t.is(options.command, 'collections-get');
+  t.is(options.collectionName, 'my-collection');
+  t.is(options.key, '*');
+});
+
+test('get with pageSize', (t) => {
+  const options = parse(
+    'collections get my-collection some-key --page-size 22'
+  );
+  t.is(options.pageSize, 22);
+});
+
+test('get with pretty output', (t) => {
+  const options = parse('collections get my-collection some-key --pretty');
+  t.is(options.pretty, true);
+});
+
+test('get with limit', (t) => {
+  const options = parse('collections get my-collection some-key --limit 999');
+  t.is(options.limit, 999);
+});
+
+test('get with output path', (t) => {
+  const options = parse(
+    'collections get my-collection some-key --o x/y/z.json'
+  );
+  t.is(options.outputPath, 'x/y/z.json');
+});
+
+test('remove with collection and key', (t) => {
+  const options = parse('collections remove my-collection some-key');
+  t.is(options.collectionName, 'my-collection');
+  t.is(options.key, 'some-key');
+});
+
+test('remove with dry run', (t) => {
+  const options = parse('collections remove my-collection some-key --dry-run');
+  t.is(options.collectionName, 'my-collection');
+  t.true(options.dryRun);
+});
+
+test('set with collection, key and value path', (t) => {
+  const options = parse('collections set my-collection some-key x/y/z.json');
+  t.is(options.collectionName, 'my-collection');
+  t.is(options.key, 'some-key');
+  t.is(options.value, 'x/y/z.json');
+});
+
+test('set with collection, key and items path', (t) => {
+  const options = parse('collections set my-collection --items x/y/z.json');
+  t.is(options.collectionName, 'my-collection');
+  t.is(options.items, 'x/y/z.json');
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 95d2fc642..2780808ee 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -231,6 +231,9 @@ importers:
         specifier: ^17.7.2
         version: 17.7.2
     devDependencies:
+      '@openfn/language-collections':
+        specifier: ^0.6.1
+        version: 0.6.1
       '@openfn/language-common':
         specifier: 2.0.0-rc3
         version: 2.0.0-rc3
@@ -1615,6 +1618,24 @@ packages:
       '@jridgewell/resolve-uri': 3.1.1
       '@jridgewell/sourcemap-codec': 1.4.15
 
+  /@jsep-plugin/assignment@1.3.0(jsep@1.4.0):
+    resolution: {integrity: sha512-VVgV+CXrhbMI3aSusQyclHkenWSAm95WaiKrMxRFam3JSUiIaQjoMIw2sEs/OX4XifnqeQUN4DYbJjlA8EfktQ==}
+    engines: {node: '>= 10.16.0'}
+    peerDependencies:
+      jsep: ^0.4.0||^1.0.0
+    dependencies:
+      jsep: 1.4.0
+    dev: true
+
+  /@jsep-plugin/regex@1.0.4(jsep@1.4.0):
+    resolution: {integrity: sha512-q7qL4Mgjs1vByCaTnDFcBnV9HS7GVPJX5vyVoCgZHNSC9rjwIlmbXG5sUuorR5ndfHAIlJ8pVStxvjXHbNvtUg==}
+    engines: {node: '>= 10.16.0'}
+    peerDependencies:
+      jsep: ^0.4.0||^1.0.0
+    dependencies:
+      jsep: 1.4.0
+    dev: true
+
   /@koa/router@12.0.0:
     resolution: {integrity: sha512-cnnxeKHXlt7XARJptflGURdJaO+ITpNkOHmQu7NHmCoRinPbyvFzce/EG/E8Zy81yQ1W9MoSdtklc3nyaDReUw==}
     engines: {node: '>= 12'}
@@ -1706,6 +1727,14 @@ packages:
     engines: {node: ^16.14.0 || >=18.0.0}
     dev: true
 
+  /@openfn/language-collections@0.6.1:
+    resolution: {integrity: sha512-CAQ5HH8o4npcux8zKHmZL+ZMVAjgvovEA76kzGu6Zaqmx/vJNH6ikv4jaTp3S0P25E9qm2U7QbtdQCVmiRHseg==}
+    dependencies:
+      '@openfn/language-common': 2.1.1
+      stream-json: 1.9.1
+      undici: 5.28.4
+    dev: true
+
   /@openfn/language-common@2.0.0-rc3:
     resolution: {integrity: sha512-7kwhBnCd1idyTB3MD9dXmUqROAhoaUIkz2AGDKuv9vn/cbZh7egEv9/PzKkRcDJYFV9qyyS+cVT3Xbgsg2ii5g==}
     bundledDependencies: []
@@ -1722,6 +1751,19 @@ packages:
       lodash: 4.17.21
       undici: 5.28.4
 
+  /@openfn/language-common@2.1.1:
+    resolution: {integrity: sha512-qIUPjdx+AIM3LW3nXhFcfnhGlgaK5np8utQuzaOSb9FYJiR5hxMFfTl1o0CPkVtUdZ/UfcTFL66cNPuEbGWabA==}
+    dependencies:
+      ajv: 8.17.1
+      csv-parse: 5.5.6
+      csvtojson: 2.0.10
+      date-fns: 2.30.0
+      http-status-codes: 2.3.0
+      jsonpath-plus: 10.2.0
+      lodash: 4.17.21
+      undici: 5.28.4
+    dev: true
+
   /@openfn/language-http@6.4.3:
     resolution: {integrity: sha512-8ihgIYId+ewMuNU9hbe5JWEWvaJInDrIEiy4EyO7tbzu5t/f1kO18JIzQWm6r7dcHiMfcG2QaXe6O3br1xOrDA==}
     dependencies:
@@ -4978,6 +5020,11 @@ packages:
     resolution: {integrity: sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==}
     dev: true
 
+  /jsep@1.4.0:
+    resolution: {integrity: sha512-B7qPcEVE3NVkmSJbaYxvv4cHkVW7DQsZz13pUMrfS8z8Q/BuShN+gcTXrUlPiGqM2/t/EEaI030bpxMqY8gMlw==}
+    engines: {node: '>= 10.16.0'}
+    dev: true
+
   /json-diff@1.0.6:
     resolution: {integrity: sha512-tcFIPRdlc35YkYdGxcamJjllUhXWv4n2rK9oJ2RsAzV4FBkuV4ojKEDgcZ+kpKxDmJKv+PFK65+1tVVOnSeEqA==}
     hasBin: true
@@ -5005,6 +5052,16 @@ packages:
     engines: {'0': node >= 0.2.0}
     dev: true
 
+  /jsonpath-plus@10.2.0:
+    resolution: {integrity: sha512-T9V+8iNYKFL2n2rF+w02LBOT2JjDnTjioaNFrxRy0Bv1y/hNsqR/EBK7Ojy2ythRHwmz2cRIls+9JitQGZC/sw==}
+    engines: {node: '>=18.0.0'}
+    hasBin: true
+    dependencies:
+      '@jsep-plugin/assignment': 1.3.0(jsep@1.4.0)
+      '@jsep-plugin/regex': 1.0.4(jsep@1.4.0)
+      jsep: 1.4.0
+    dev: true
+
   /jsonpath-plus@4.0.0:
     resolution: {integrity: sha512-e0Jtg4KAzDJKKwzbLaUtinCn0RZseWBVRTRGihSpvFlM3wTR7ExSp+PTdeTsDrLNJUe7L7JYJe8mblHX5SCT6A==}
     engines: {node: '>=10.0'}
@@ -6748,6 +6805,16 @@ packages:
     engines: {node: '>= 0.8'}
     dev: false
 
+  /stream-chain@2.2.5:
+    resolution: {integrity: sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==}
+    dev: true
+
+  /stream-json@1.9.1:
+    resolution: {integrity: sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==}
+    dependencies:
+      stream-chain: 2.2.5
+    dev: true
+
   /stream-shift@1.0.3:
     resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}
     dev: true

From 0c053c83243aa125973539ca4f2ca6a270500561 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sat, 7 Dec 2024 15:58:27 +0000
Subject: [PATCH 10/23] cli: collections unit tests

---
 packages/cli/src/collections/command.ts       |   3 +-
 packages/cli/src/collections/handler.ts       |   2 +-
 packages/cli/src/collections/request.ts       |  18 +-
 .../cli/test/collections/collections.test.ts  | 171 ++++++++++++++++--
 4 files changed, 168 insertions(+), 26 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 893667a66..ecbfe9ef2 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -74,7 +74,8 @@ const token = {
 const lightningUrl = {
   name: 'lightning',
   yargs: {
-    description: 'URL to Lightning server',
+    description:
+      'URL to OpenFn server. Defaults to OPENFN_ENDPOINT or https://app.openfn.org',
   },
 };
 
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 9781dacb1..0995cbf0e 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -157,7 +157,7 @@ export const remove = async (options: RemoveOptions, logger: Logger) => {
       },
       logger
     );
-    const keys = Object.keys(result.items);
+    const keys = Object.keys(result);
     logger.info('Keys to be removed:');
     logger.print(keys);
 
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index f14caa193..7579f39a3 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -24,22 +24,16 @@ export default async (
   options: Options,
   logger: Logger
 ) => {
-  // if (!state.configuration.collections_token) {
-  //   throwError('INVALID_AUTH', {
-  //     description: 'No access key provided for collection request',
-  //     fix: 'Ensure the "collections_token" value is set on state.configuration',
-  //     path,
-  //   });
-  // }
-
   const base =
-    options.lightning || 'http://localhost:4000' || 'https://app.openfn.org';
+    options.lightning ||
+    process.env.OPENFN_ENDPOINT ||
+    'https://app.openfn.org';
 
   const url = path.join(base, '/collections', options.collectionName);
 
   logger.debug('Calling Collections server at ', url);
 
-  const headers = {
+  const headers: Record<string, string> = {
     Authorization: `Bearer ${options.token}`,
   };
 
@@ -56,7 +50,7 @@ export default async (
 
   if (options.data) {
     args.body = JSON.stringify(options.data);
-    args.headers['content-type'] = 'application/json';
+    headers['content-type'] = 'application/json';
   }
 
   let result: any = {};
@@ -64,7 +58,7 @@ export default async (
   let cursor;
   do {
     if (cursor) {
-      args.query.cursor = cursor;
+      query.cursor = cursor;
     }
 
     const response = await request(url, args);
diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
index 1fad94761..6794235cf 100644
--- a/packages/cli/test/collections/collections.test.ts
+++ b/packages/cli/test/collections/collections.test.ts
@@ -8,13 +8,13 @@ import { get, set, remove } from '../../src/collections/handler';
 import { setGlobalDispatcher } from 'undici';
 import { createMockLogger } from '@openfn/logger';
 import { collections } from '@openfn/language-collections';
-import { lightning } from '@openfn/lexicon';
 import { readFile } from 'fs/promises';
 
 // Log as json to make testing easier
 const logger = createMockLogger('default', { level: 'debug', json: true });
 
 const COLLECTION = 'test-collection-a';
+const ENDPOINT = 'https://mock.openfn.org';
 
 let api: any;
 
@@ -34,7 +34,7 @@ const loadData = (items: Record<string, object>) => {
 };
 
 test.before(() => {
-  const client = collections.createMockServer('https://mock.openfn.org');
+  const client = collections.createMockServer(ENDPOINT);
   api = client.api;
   setGlobalDispatcher(client.agent);
 });
@@ -43,29 +43,33 @@ test.beforeEach(() => {
   logger._reset();
   api.reset();
   api.createCollection(COLLECTION);
+  loadData({
+    x: {},
+    y: {},
+  });
+
   resetMockFs();
 });
 
 const createOptions = (opts = {}) => ({
-  lightning: 'https://mock.openfn.org',
+  lightning: ENDPOINT,
   collectionName: COLLECTION,
   key: '*',
+  token: 'x.y.z', // TODO need more tests around this
   ...opts,
 });
 
 test.serial(
   'should get all keys from a collection and print to stdout',
   async (t) => {
-    loadData({
-      x: {},
-      y: {},
+    const options = createOptions({
+      key: '*',
     });
-    const options = createOptions();
     await get(options, logger);
 
     // The last log should print the data out
     // (because we're logging to JSON we can easily inspect the raw JSON data)
-    const [level, log] = logger._history.at(-1);
+    const [_level, log] = logger._history.at(-1);
     t.deepEqual(log.message[0], {
       x: { id: 'x' },
       y: { id: 'y' },
@@ -73,17 +77,31 @@ test.serial(
   }
 );
 
+test.serial(
+  'should get one key from a collection and print to stdout',
+  async (t) => {
+    const options = createOptions({
+      key: 'x',
+    });
+    await get(options, logger);
+
+    // The last log should print the data out
+    // (because we're logging to JSON we can easily inspect the raw JSON data)
+    const [_level, log] = logger._history.at(-1);
+    t.deepEqual(log.message[0], {
+      id: 'x',
+    });
+  }
+);
+
 test.serial(
   'should get all keys from a collection and write to disk',
   async (t) => {
     mockFs({
       '/tmp.json': '',
     });
-    loadData({
-      x: {},
-      y: {},
-    });
     const options = createOptions({
+      key: '*',
       outputPath: '/tmp.json',
     });
     await get(options, logger);
@@ -97,3 +115,132 @@ test.serial(
     });
   }
 );
+
+test.serial(
+  'should get one key from a collection and write to disk',
+  async (t) => {
+    mockFs({
+      '/tmp.json': '',
+    });
+    const options = createOptions({
+      key: 'x',
+      outputPath: '/tmp.json',
+    });
+    await get(options, logger);
+
+    const data = await readFile('/tmp.json');
+    const items = JSON.parse(data);
+
+    t.deepEqual(items, {
+      id: 'x',
+    });
+  }
+);
+
+// TODO collection doesn't exist
+// TODO item doesn't exist
+// TODO no matching values
+
+test.serial(
+  'should use OPENFN_ENDPOINT if lightning option is not set',
+  async (t) => {
+    const options = createOptions({
+      key: 'x',
+      lightning: undefined,
+    });
+    process.env.OPENFN_ENDPOINT = ENDPOINT;
+
+    await get(options, logger);
+
+    const [_level, log] = logger._history.at(-1);
+    t.deepEqual(log.message[0], {
+      id: 'x',
+    });
+
+    delete process.env.OPENFN_ENDPOINT;
+  }
+);
+
+// TODO test that limit actually works
+// TODO test that query filters actually work (mock doesn't support this)
+
+test.serial('should set a single value', async (t) => {
+  mockFs({
+    '/value.json': JSON.stringify({ id: 'z' }),
+  });
+  const options = createOptions({
+    key: 'z',
+    value: '/value.json',
+  });
+
+  await set(options, logger);
+
+  t.is(api.count(COLLECTION), 3);
+  const item = api.asJSON(COLLECTION, 'z');
+  t.deepEqual(item, { id: 'z' });
+});
+
+test.serial('should set multiple values', async (t) => {
+  mockFs({
+    '/items.json': JSON.stringify({
+      a: { id: 'a' },
+      b: { id: 'b' },
+    }),
+  });
+  const options = createOptions({
+    key: 'z',
+    items: '/items.json',
+  });
+
+  await set(options, logger);
+
+  t.is(api.count(COLLECTION), 4);
+  const a = api.asJSON(COLLECTION, 'a');
+  t.deepEqual(a, { id: 'a' });
+
+  const b = api.asJSON(COLLECTION, 'b');
+  t.deepEqual(b, { id: 'b' });
+});
+
+test.serial('should remove one key', async (t) => {
+  const itemBefore = api.byKey(COLLECTION, 'x');
+  t.truthy(itemBefore);
+
+  const options = createOptions({
+    key: 'x',
+  });
+
+  await remove(options, logger);
+
+  const itemAfter = api.byKey(COLLECTION, 'x');
+  t.falsy(itemAfter);
+});
+
+test.serial('should remove multiple keys', async (t) => {
+  t.is(api.count(COLLECTION), 2);
+
+  const options = createOptions({
+    key: '*',
+  });
+
+  await remove(options, logger);
+
+  t.is(api.count(COLLECTION), 0);
+});
+
+test.serial('should do a dry run', async (t) => {
+  t.is(api.count(COLLECTION), 2);
+
+  const options = createOptions({
+    key: '*',
+    dryRun: true,
+  });
+
+  await remove(options, logger);
+
+  t.is(api.count(COLLECTION), 2);
+
+  // Find the outputted keys
+  const [_level, output] = logger._history.find(([level]) => level === 'print');
+  t.deepEqual(output.message[0], ['x', 'y']);
+});

From 89636dd95e2635707e1e1792c6d241b2e9886625 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sat, 7 Dec 2024 17:28:23 +0000
Subject: [PATCH 11/23] cli: collections error handling and tests

---
 packages/cli/src/collections/request.ts       | 97 +++++++++++++------
 packages/cli/src/commands.ts                  |  8 +-
 packages/cli/src/util/abort.ts                | 17 ++++
 .../cli/test/collections/collections.test.ts  | 30 +++++-
 4 files changed, 123 insertions(+), 29 deletions(-)

diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 7579f39a3..5f98a4908 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -2,6 +2,7 @@ import path from 'node:path';
 import { request } from 'undici';
 import type { Dispatcher } from 'undici';
 import { Logger } from '../util';
+import abort, { throwAbortableError } from '../util/abort';
 
 // helper function to call out to the collections API
 
@@ -61,38 +62,80 @@ export default async (
       query.cursor = cursor;
     }
 
-    const response = await request(url, args);
-    if (response.statusCode >= 400) {
-      // await handleError(response, path, state.configuration.collections_endpoint);
-      logger.error('error!');
-    }
-    const responseData: any = await response.body.json();
-
-    if (responseData.items) {
-      // Handle a get response
-      logger.debug(
-        'Received',
-        response.statusCode,
-        `- ${responseData.items.length} values`
-      );
-      for (const item of responseData?.items) {
-        try {
-          result[item.key] = JSON.parse(item.value);
-        } catch (e) {
-          result[item.key] = item.value;
+    try {
+      const response = await request(url, args);
+
+      if (response.statusCode >= 400) {
+        return handleError(logger, response);
+      }
+      const responseData: any = await response.body.json();
+
+      if (responseData.items) {
+        // Handle a get response
+        logger.debug(
+          'Received',
+          response.statusCode,
+          `- ${responseData.items.length} values`
+        );
+        for (const item of responseData?.items) {
+          try {
+            result[item.key] = JSON.parse(item.value);
+          } catch (e) {
+            result[item.key] = item.value;
+          }
         }
+        cursor = responseData.cursor;
+      } else {
+        // handle a set response
+        logger.debug(
+          'Received',
+          response.statusCode,
+          `- ${JSON.stringify(responseData)}`
+        );
+        result = responseData;
       }
-      cursor = responseData.cursor;
-    } else {
-      // handle a set response
-      logger.debug(
-        'Received',
-        response.statusCode,
-        `- ${JSON.stringify(responseData)}`
+    } catch (e: any) {
+      logger.error(e);
+      throwAbortableError(
+        `CONNECTION_REFUSED: error connecting to server at ${base}`,
+        'Check you have passed the correct URL to --lightning or OPENFN_ENDPOINT'
       );
-      result = responseData;
     }
   } while (cursor);
 
   return result;
 };
+
+async function handleError(
+  logger: Logger,
+  response: Dispatcher.ResponseData<any>
+) {
+  logger.error('Error from server', response.statusCode);
+  let message;
+  let fix;
+
+  switch (response.statusCode) {
+    case 404:
+      message = `404: collection not found`;
+      fix = `Ensure the Collection has been created on the admin page`;
+      break;
+    default:
+      message = `Error from server: ${response.statusCode}`;
+  }
+
+  let contentType = (response.headers?.['content-type'] as string) ?? '';
+
+  if (contentType.startsWith('application/json')) {
+    try {
+      const body = await response.body.json();
+      logger.error(body);
+    } catch (e) {}
+  } else {
+    try {
+      const text = await response.body.text();
+      logger.error(text);
+    } catch (e) {}
+  }
+
+  throwAbortableError(message, fix);
+}
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index 1cadfd5f9..29127bb59 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -16,6 +16,7 @@ import mapAdaptorsToMonorepo, {
   validateMonoRepo,
 } from './util/map-adaptors-to-monorepo';
 import printVersions from './util/print-versions';
+import abort from './util/abort';
 
 export type CommandList =
   | 'apollo'
@@ -123,7 +124,12 @@ const parse = async (options: Opts, log?: Logger) => {
       process.exitCode = e.exitCode || 1;
     }
     if (e.handled) {
-      // If throwing an epected error from util/abort, we do nothing
+      // If throwing an expected error from util/abort, we do nothing
+    } else if (e.abort) {
+      try {
+        // Run the abort code but catch the error
+        abort(logger, e.reason, e.error, e.help);
+      } catch (e) {}
     } else {
       // This is unexpected error and we should try to log something
       logger.break();
diff --git a/packages/cli/src/util/abort.ts b/packages/cli/src/util/abort.ts
index 1644f110a..23adaede4 100644
--- a/packages/cli/src/util/abort.ts
+++ b/packages/cli/src/util/abort.ts
@@ -30,3 +30,20 @@ export default (
 
   throw e;
 };
+
+class DeferredAbort extends Error {
+  constructor(reason: string, help?: string) {
+    super('DeferredAbortError');
+    this.reason = reason;
+    this.help = help ?? '';
+  }
+  abort = true;
+  reason = '';
+  help = '';
+}
+// This function lets us create an error that can be aborted
+// but the top level command handler, resulting in code
+// that's easier to test
+export const throwAbortableError = (message: string, help?: string) => {
+  throw new DeferredAbort(message, help);
+};
diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
index 6794235cf..af3232b02 100644
--- a/packages/cli/test/collections/collections.test.ts
+++ b/packages/cli/test/collections/collections.test.ts
@@ -9,6 +9,7 @@ import { setGlobalDispatcher } from 'undici';
 import { createMockLogger } from '@openfn/logger';
 import { collections } from '@openfn/language-collections';
 import { readFile } from 'fs/promises';
+import { lightning } from '@openfn/lexicon';
 
 // Log as json to make testing easier
 const logger = createMockLogger('default', { level: 'debug', json: true });
@@ -137,7 +138,6 @@ test.serial(
   }
 );
 
-// TODO collection doesn't exist
 // TODO item doesn't exist
 // TODO no matching values
 
@@ -244,3 +244,31 @@ test.serial('should do a dry run', async (t) => {
   const [_level, output] = logger._history.find(([level]) => level === 'print');
   t.deepEqual(output.message[0], ['x', 'y']);
 });
+
+// These tests are against the request helper code and should be common to all verbs
+
+test.serial('should throw if the server is not available', async (t) => {
+  const options = createOptions({
+    key: 'x',
+    lightning: 'https://www.blah.blah.blah',
+  });
+  try {
+    await get(options, logger);
+  } catch (e: any) {
+    t.regex(e.reason, /connection_refused/i);
+    t.regex(e.help, /correct url .+ --lightning/i);
+  }
+});
+
+test.serial("should throw if a collection doesn't exist", async (t) => {
+  const options = createOptions({
+    key: 'x',
+    collectionName: 'strawberries',
+  });
+  try {
+    await get(options, logger);
+  } catch (e: any) {
+    t.regex(e.reason, /collection not found/i);
+    t.regex(e.help, /ensure the collection has been created/i);
+  }
+});

From 5cacdbe34e5a39e7b2a3e6066c99295c9da341b7 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sat, 7 Dec 2024 17:29:53 +0000
Subject: [PATCH 12/23] cli: typings

---
 packages/cli/src/collections/handler.ts | 8 ++++----
 packages/cli/src/collections/request.ts | 5 ++---
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 0995cbf0e..c11f2d649 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -52,7 +52,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
     'GET',
     {
       lightning: options.lightning,
-      token: options.token,
+      token: options.token!,
       pageSize: options.pageSize,
       limit: options.limit,
       key: options.key,
@@ -123,7 +123,7 @@ export const set = async (options: SetOptions, logger: Logger) => {
     'POST',
     {
       lightning: options.lightning,
-      token: options.token,
+      token: options.token!,
       key: options.key,
       collectionName: options.collectionName,
       data: { items },
@@ -151,7 +151,7 @@ export const remove = async (options: RemoveOptions, logger: Logger) => {
       'GET',
       {
         lightning: options.lightning,
-        token: options.token,
+        token: options.token!,
         key: options.key,
         collectionName: options.collectionName,
       },
@@ -167,7 +167,7 @@ export const remove = async (options: RemoveOptions, logger: Logger) => {
       'DELETE',
       {
         lightning: options.lightning,
-        token: options.token,
+        token: options.token!,
         key: options.key,
         collectionName: options.collectionName,
       },
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 5f98a4908..27c422f46 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -2,12 +2,10 @@ import path from 'node:path';
 import { request } from 'undici';
 import type { Dispatcher } from 'undici';
 import { Logger } from '../util';
-import abort, { throwAbortableError } from '../util/abort';
+import { throwAbortableError } from '../util/abort';
 
 // helper function to call out to the collections API
 
-// export const request = async (state, client, path, options = {}) => {
-
 type Options = {
   key: string;
   collectionName: string;
@@ -16,6 +14,7 @@ type Options = {
 
   includeMeta?: boolean; // TODO ignored right now
   pageSize?: number;
+  limit?: number;
 
   data?: any;
 };

From ec53eee456ebe52b3c0c14fe9314b2447606f8e4 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sun, 8 Dec 2024 12:34:06 +0000
Subject: [PATCH 13/23] bump collections adaptor version

---
 packages/cli/package.json | 2 +-
 pnpm-lock.yaml            | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/cli/package.json b/packages/cli/package.json
index a578c7218..8bcefe462 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -33,7 +33,7 @@
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",
   "devDependencies": {
-    "@openfn/language-collections": "^0.6.1",
+    "@openfn/language-collections": "^0.6.2",
     "@openfn/language-common": "2.0.0-rc3",
     "@openfn/lexicon": "workspace:^",
     "@types/mock-fs": "^4.13.1",
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2780808ee..d2bb1d213 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -232,8 +232,8 @@ importers:
         version: 17.7.2
     devDependencies:
       '@openfn/language-collections':
-        specifier: ^0.6.1
-        version: 0.6.1
+        specifier: ^0.6.2
+        version: 0.6.2
       '@openfn/language-common':
         specifier: 2.0.0-rc3
         version: 2.0.0-rc3
@@ -1727,8 +1727,8 @@ packages:
     engines: {node: ^16.14.0 || >=18.0.0}
     dev: true
 
-  /@openfn/language-collections@0.6.1:
-    resolution: {integrity: sha512-CAQ5HH8o4npcux8zKHmZL+ZMVAjgvovEA76kzGu6Zaqmx/vJNH6ikv4jaTp3S0P25E9qm2U7QbtdQCVmiRHseg==}
+  /@openfn/language-collections@0.6.2:
+    resolution: {integrity: sha512-EyXuXvYGBmBXgF95snuxWCd+HgZsT57ghqzDUnhYC+qaUNe9p0aIlFdfA2tTokce04KWI8hc7HKnvm0yPd5H7A==}
     dependencies:
       '@openfn/language-common': 2.1.1
       stream-json: 1.9.1

From 98a2ed746f15bc2b1016e32129ad98fd08c7efcd Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Sun, 8 Dec 2024 17:47:58 +0000
Subject: [PATCH 14/23] cli: better error handling in collections

---
 packages/cli/src/collections/command.ts       |  18 +-
 packages/cli/src/collections/handler.ts       |   8 +
 .../cli/test/collections/collections.test.ts  | 190 +++++++++---------
 3 files changed, 117 insertions(+), 99 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index ecbfe9ef2..98051e277 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -26,7 +26,7 @@ export type SetOptions = CollectionsOptions & {
   value?: string;
 };
 
-const desc = `Call out to the Collections API on Lightning`;
+const desc = `Read and write from the OpenFn Collections API`;
 
 export default {
   command: 'collections <subcommand>',
@@ -37,8 +37,20 @@ export default {
       .command(set)
       .command(remove)
       .example(
-        'collections get my-collection 2024* -O',
-        'Get all keys from my-collection starting with the string "2024" and log the results to stdout'
+        '$0 collections get my-collection 2024* -o /tmp/output.json',
+        'Get all keys from my-collection starting with the string "2024" and output the results to file'
+      )
+      .example(
+        '$0 collections set my-collection my-key path/to/value.json',
+        'Set a single key in my-collection to the contents of value.json'
+      )
+      .example(
+        '$0 collections set my-collection --items path/to/items.json',
+        'Set multiple key/value pairs from items.json to my-collection'
+      )
+      .example(
+        '$0 collections remove my-collection my-key',
+        'Remove a single key from my-collection'
       ),
 } as yargs.CommandModule<{}>;
 
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index c11f2d649..2e28ae756 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -10,6 +10,7 @@ import type {
   RemoveOptions,
   SetOptions,
 } from './command';
+import { throwAbortableError } from '../util/abort';
 
 const ensureToken = (opts: CollectionsOptions, logger: Logger) => {
   if (!('token' in opts)) {
@@ -83,6 +84,13 @@ export const get = async (options: GetOptions, logger: Logger) => {
 };
 
 export const set = async (options: SetOptions, logger: Logger) => {
+  if (options.key && options.items) {
+    throwAbortableError(
+      'ARGUMENT_ERROR: arguments for key and items were provided',
+      'If upserting multiple items with --items, do not pass a key'
+    );
+  }
+
   ensureToken(options, logger);
   logger.info(`Upserting items to collection "${options.collectionName}"`);
 
diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
index af3232b02..7e167f157 100644
--- a/packages/cli/test/collections/collections.test.ts
+++ b/packages/cli/test/collections/collections.test.ts
@@ -60,111 +60,79 @@ const createOptions = (opts = {}) => ({
   ...opts,
 });
 
-test.serial(
-  'should get all keys from a collection and print to stdout',
-  async (t) => {
-    const options = createOptions({
-      key: '*',
-    });
-    await get(options, logger);
-
-    // The last log should print the data out
-    // (because we're logging to JSON we can easily inspect the raw JSON data)
-    const [_level, log] = logger._history.at(-1);
-    t.deepEqual(log.message[0], {
-      x: { id: 'x' },
-      y: { id: 'y' },
-    });
-  }
-);
-
-test.serial(
-  'should get one key from a collection and print to stdout',
-  async (t) => {
-    const options = createOptions({
-      key: 'x',
-    });
-    await get(options, logger);
-
-    // The last log should print the data out
-    // (because we're logging to JSON we can easily inspect the raw JSON data)
-    const [_level, log] = logger._history.at(-1);
-    t.deepEqual(log.message[0], {
-      id: 'x',
-    });
-  }
-);
-
-test.serial(
-  'should get all keys from a collection and write to disk',
-  async (t) => {
-    mockFs({
-      '/tmp.json': '',
-    });
-    const options = createOptions({
-      key: '*',
-      outputPath: '/tmp.json',
-    });
-    await get(options, logger);
+test.serial('get all keys from a collection and print to stdout', async (t) => {
+  const options = createOptions({
+    key: '*',
+  });
+  await get(options, logger);
+
+  // The last log should print the data out
+  // (because we're logging to JSON we can easily inspect the raw JSON data)
+  const [_level, log] = logger._history.at(-1);
+  t.deepEqual(log.message[0], {
+    x: { id: 'x' },
+    y: { id: 'y' },
+  });
+});
 
-    const data = await readFile('/tmp.json');
-    const items = JSON.parse(data);
+test.serial('get one key from a collection and print to stdout', async (t) => {
+  const options = createOptions({
+    key: 'x',
+  });
+  await get(options, logger);
 
-    t.deepEqual(items, {
-      x: { id: 'x' },
-      y: { id: 'y' },
-    });
-  }
-);
-
-test.serial(
-  'should get one key from a collection and write to disk',
-  async (t) => {
-    mockFs({
-      '/tmp.json': '',
-    });
-    const options = createOptions({
-      key: 'x',
-      outputPath: '/tmp.json',
-    });
-    await get(options, logger);
+  // The last log should print the data out
+  // (because we're logging to JSON we can easily inspect the raw JSON data)
+  const [_level, log] = logger._history.at(-1);
+  t.deepEqual(log.message[0], {
+    id: 'x',
+  });
+});
 
-    const data = await readFile('/tmp.json');
-    const items = JSON.parse(data);
+test.serial('get all keys from a collection and write to disk', async (t) => {
+  mockFs({
+    '/tmp.json': '',
+  });
+  const options = createOptions({
+    key: '*',
+    outputPath: '/tmp.json',
+  });
+  await get(options, logger);
 
-    t.deepEqual(items, {
-      id: 'x',
-    });
-  }
-);
+  const data = await readFile('/tmp.json');
+  const items = JSON.parse(data);
 
-// TODO item doesn't exist
-// TODO no matching values
+  t.deepEqual(items, {
+    x: { id: 'x' },
+    y: { id: 'y' },
+  });
+});
 
-test.serial(
-  'should use OPENFN_ENDPOINT if lightning option is not set',
-  async (t) => {
-    const options = createOptions({
-      key: 'x',
-      lightning: undefined,
-    });
-    process.env.OPENFN_ENDPOINT = ENDPOINT;
+test.serial('get one key from a collection and write to disk', async (t) => {
+  mockFs({
+    '/tmp.json': '',
+  });
+  const options = createOptions({
+    key: 'x',
+    outputPath: '/tmp.json',
+  });
+  await get(options, logger);
 
-    await get(options, logger);
+  const data = await readFile('/tmp.json');
+  const items = JSON.parse(data);
 
-    const [_level, log] = logger._history.at(-1);
-    t.deepEqual(log.message[0], {
-      id: 'x',
-    });
+  t.deepEqual(items, {
+    id: 'x',
+  });
+});
 
-    delete process.env.OPENFN_ENDPOINT;
-  }
-);
+// TODO item doesn't exist
+// TODO no matching values
 
 // TODO test that limit actually works
 // TODO test that query filters actually work (mock doesn't support this)
 
-test.serial('should set a single value', async (t) => {
+test.serial('set a single value', async (t) => {
   mockFs({
     '/value.json': JSON.stringify({ id: 'z' }),
   });
@@ -180,7 +148,7 @@ test.serial('should set a single value', async (t) => {
   t.deepEqual(item, { id: 'z' });
 });
 
-test.serial('should set multiple values', async (t) => {
+test.serial('set multiple values', async (t) => {
   mockFs({
     '/items.json': JSON.stringify({
       a: { id: 'a' },
@@ -202,7 +170,20 @@ test.serial('should set multiple values', async (t) => {
   t.deepEqual(b, { id: 'b' });
 });
 
-test.serial('should remove one key', async (t) => {
+test.serial('set should throw if key and items are both set', async (t) => {
+  const options = createOptions({
+    key: 'z',
+    items: '/value.json',
+  });
+  try {
+    await set(options, logger);
+  } catch (e) {
+    t.regex(e.reason, /argument_error/i);
+    t.regex(e.help, /do not pass a key/i);
+  }
+});
+
+test.serial('remove one key', async (t) => {
   const itemBefore = api.byKey(COLLECTION, 'x');
   t.truthy(itemBefore);
 
@@ -216,7 +197,7 @@ test.serial('should remove one key', async (t) => {
   t.falsy(itemAfter);
 });
 
-test.serial('should remove multiple keys', async (t) => {
+test.serial('remove multiple keys', async (t) => {
   t.is(api.count(COLLECTION), 2);
 
   const options = createOptions({
@@ -228,7 +209,7 @@ test.serial('should remove multiple keys', async (t) => {
   t.is(api.count(COLLECTION), 0);
 });
 
-test.serial('should do a dry run', async (t) => {
+test.serial('remove with dry run', async (t) => {
   t.is(api.count(COLLECTION), 2);
 
   const options = createOptions({
@@ -272,3 +253,20 @@ test.serial("should throw if a collection doesn't exist", async (t) => {
     t.regex(e.help, /ensure the collection has been created/i);
   }
 });
+
+test.serial('use OPENFN_ENDPOINT if lightning option is not set', async (t) => {
+  const options = createOptions({
+    key: 'x',
+    lightning: undefined,
+  });
+  process.env.OPENFN_ENDPOINT = ENDPOINT;
+
+  await get(options, logger);
+
+  const [_level, log] = logger._history.at(-1);
+  t.deepEqual(log.message[0], {
+    id: 'x',
+  });
+
+  delete process.env.OPENFN_ENDPOINT;
+});

From ae2ae9331cbc417f537428188e4553ffe31b906e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Mon, 9 Dec 2024 09:36:13 +0000
Subject: [PATCH 15/23] cli: fix collections test

---
 packages/cli/test/collections/collections.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
index 7e167f157..e385a0603 100644
--- a/packages/cli/test/collections/collections.test.ts
+++ b/packages/cli/test/collections/collections.test.ts
@@ -156,8 +156,8 @@ test.serial('set multiple values', async (t) => {
     }),
   });
   const options = createOptions({
-    key: 'z',
     items: '/items.json',
+    key: null,
   });
 
   await set(options, logger);

From 82c06615e919063a6167268eafadd2b40a435026 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Mon, 9 Dec 2024 16:37:31 +0000
Subject: [PATCH 16/23] collections cli: support limit

---
 packages/cli/src/collections/handler.ts       |  2 +-
 packages/cli/src/collections/request.ts       | 20 ++++-
 .../cli/test/collections/collections.test.ts  | 79 ++++++++++++++++++-
 3 files changed, 96 insertions(+), 5 deletions(-)

diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 2e28ae756..495e87e5a 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -63,7 +63,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
   );
 
   if (multiMode) {
-    logger.success(`Fetched ${result.count} items!`);
+    logger.success(`Fetched ${Object.keys(result).length} items!`);
   } else {
     result = Object.values(result)[0];
     logger.success(`Fetched ${options.key}`);
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index 27c422f46..e48b5a815 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -19,6 +19,8 @@ type Options = {
   data?: any;
 };
 
+const DEFAULT_PAGE_SIZE = 1000;
+
 export default async (
   method: 'GET' | 'POST' | 'DELETE',
   options: Options,
@@ -39,7 +41,7 @@ export default async (
 
   const query: any = {
     key: options.key,
-    limit: options.pageSize || 1000,
+    limit: options.pageSize || DEFAULT_PAGE_SIZE,
   };
 
   const args: Partial<Dispatcher.RequestOptions> = {
@@ -54,13 +56,24 @@ export default async (
   }
 
   let result: any = {};
-
   let cursor;
+  let count = 0;
+  let limit = Infinity;
+
   do {
     if (cursor) {
       query.cursor = cursor;
     }
 
+    if (options.limit) {
+      limit = options.limit;
+      // Make sure the next page size respects the user limit
+      query.limit = Math.min(
+        options.pageSize || DEFAULT_PAGE_SIZE,
+        options.limit - count
+      );
+    }
+
     try {
       const response = await request(url, args);
 
@@ -70,6 +83,7 @@ export default async (
       const responseData: any = await response.body.json();
 
       if (responseData.items) {
+        count += responseData.items.length;
         // Handle a get response
         logger.debug(
           'Received',
@@ -100,7 +114,7 @@ export default async (
         'Check you have passed the correct URL to --lightning or OPENFN_ENDPOINT'
       );
     }
-  } while (cursor);
+  } while (cursor && count < limit);
 
   return result;
 };
diff --git a/packages/cli/test/collections/collections.test.ts b/packages/cli/test/collections/collections.test.ts
index e385a0603..94ccca9bc 100644
--- a/packages/cli/test/collections/collections.test.ts
+++ b/packages/cli/test/collections/collections.test.ts
@@ -9,7 +9,6 @@ import { setGlobalDispatcher } from 'undici';
 import { createMockLogger } from '@openfn/logger';
 import { collections } from '@openfn/language-collections';
 import { readFile } from 'fs/promises';
-import { lightning } from '@openfn/lexicon';
 
 // Log as json to make testing easier
 const logger = createMockLogger('default', { level: 'debug', json: true });
@@ -126,6 +125,84 @@ test.serial('get one key from a collection and write to disk', async (t) => {
   });
 });
 
+// We don't have a great way to test pagination because it's
+// all internal. We can't even get into the mock from here.
+// So we'll test on the logs - it's good enough for today
+test.serial('get 200 items over 4 pages', async (t) => {
+  api.reset();
+  api.createCollection(COLLECTION);
+
+  new Array(300).fill(0).forEach((_v, idx) => {
+    api.upsert(
+      COLLECTION,
+      idx,
+      JSON.stringify({
+        id: `${idx}`,
+        score: idx,
+      })
+    );
+  });
+  t.is(api.count(COLLECTION), 300);
+
+  const options = createOptions({
+    key: '*',
+    pageSize: 50,
+    limit: 200,
+  });
+
+  await get(options, logger);
+
+  const findLogs = logger._history.filter(
+    (l) => l.message && l.message.join(' ') === 'Received 200 - 50 values'
+  );
+  t.is(findLogs.length, 4);
+
+  // Find the stdout call
+  const [_level, log] = logger._history.at(-1);
+  const result = log.message[0];
+
+  // Should be 200 items returned
+  t.is(Object.keys(result).length, 200);
+});
+
+// This test uses an irregular page size
+test.serial('get 180 items over 4 pages', async (t) => {
+  api.reset();
+  api.createCollection(COLLECTION);
+
+  new Array(300).fill(0).forEach((_v, idx) => {
+    api.upsert(
+      COLLECTION,
+      idx,
+      JSON.stringify({
+        id: `${idx}`,
+        score: idx,
+      })
+    );
+  });
+  t.is(api.count(COLLECTION), 300);
+
+  const options = createOptions({
+    key: '*',
+    pageSize: 50,
+    limit: 180,
+  });
+
+  await get(options, logger);
+
+  const findLogs = logger._history.filter(
+    (l) => l.message && l.message.join(' ').match(/Received 200 - \d\d values/i)
+  );
+  t.is(findLogs.length, 4);
+
+  // Find the stdout call
+  const [_level, log] = logger._history.at(-1);
+  const result = log.message[0];
+
+  // Should be 200 items returned
+  t.is(Object.keys(result).length, 180);
+});
+
 // TODO item doesn't exist
 // TODO no matching values
 

From ba131cb3e664118fc1baae0c3a79d3c04573036e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Mon, 9 Dec 2024 17:16:18 +0000
Subject: [PATCH 17/23] cli: hook up collections queries

No unit tests becuase we don't have mock support yet
---
 packages/cli/src/collections/command.ts | 52 +++++++++++++++++++++++--
 packages/cli/src/collections/handler.ts | 19 +++++++++
 packages/cli/src/collections/request.ts | 13 +++++--
 3 files changed, 77 insertions(+), 7 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 98051e277..385d364ed 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -3,6 +3,13 @@ import * as o from '../options';
 import type { Opts } from '../options';
 import { build, ensure, override } from '../util/command-builders';
 
+export type QueryOptions = {
+  createdBefore?: string;
+  createdAfter?: string;
+  updatedBefore?: string;
+  updatedAfter?: string;
+};
+
 export type CollectionsOptions = Pick<Opts, 'log' | 'logJson'> & {
   lightning?: string;
   token?: string;
@@ -11,15 +18,17 @@ export type CollectionsOptions = Pick<Opts, 'log' | 'logJson'> & {
 };
 
 export type GetOptions = CollectionsOptions &
+  QueryOptions &
   Pick<Opts, 'outputPath' | 'outputStdout'> & {
     pageSize?: number;
     limit?: number;
     pretty?: boolean;
   };
 
-export type RemoveOptions = CollectionsOptions & {
-  dryRun?: boolean;
-};
+export type RemoveOptions = CollectionsOptions &
+  QueryOptions & {
+    dryRun?: boolean;
+  };
 
 export type SetOptions = CollectionsOptions & {
   items?: string;
@@ -116,6 +125,33 @@ const pretty = {
   },
 };
 
+const createdBefore = {
+  name: 'created-before',
+  yargs: {
+    description: 'Matches keys created before the start of the created data',
+  },
+};
+
+const createdAfter = {
+  name: 'created-after',
+  yargs: {
+    description: 'Matches keys created after the end of the created data',
+  },
+};
+const updatedBefore = {
+  name: 'updated-before',
+  yargs: {
+    description: 'Matches keys updated before the start of the created data',
+  },
+};
+
+const updatedAfter = {
+  name: 'updated-after',
+  yargs: {
+    description: 'Matches keys updated after the end of the created data',
+  },
+};
+
 const getOptions = [
   collectionName,
   key,
@@ -125,6 +161,11 @@ const getOptions = [
   limit,
   pretty,
 
+  createdBefore,
+  createdAfter,
+  updatedAfter,
+  updatedBefore,
+
   override(o.log, {
     default: 'info',
   }),
@@ -159,6 +200,11 @@ const removeOptions = [
   lightningUrl,
   dryRun,
 
+  createdBefore,
+  createdAfter,
+  updatedAfter,
+  updatedBefore,
+
   override(o.log, {
     default: 'info',
   }),
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 495e87e5a..3e421d289 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -11,6 +11,7 @@ import type {
   SetOptions,
 } from './command';
 import { throwAbortableError } from '../util/abort';
+import { QueryOptions } from '@openfn/language-collections/types/collections';
 
 const ensureToken = (opts: CollectionsOptions, logger: Logger) => {
   if (!('token' in opts)) {
@@ -36,6 +37,22 @@ const ensureToken = (opts: CollectionsOptions, logger: Logger) => {
   }
 };
 
+const buildQuery = (options: any) => {
+  const map: Record<keyof QueryOptions, string> = {
+    createdBefore: 'created_before',
+    createdAfter: 'created_after',
+    updatedBefore: 'updated_before',
+    updatedAfter: 'updated_after',
+  };
+  const query: any = {};
+  Object.keys(map).forEach((key) => {
+    if (options[key]) {
+      query[map[key]] = options[key];
+    }
+  });
+  return query;
+};
+
 export const get = async (options: GetOptions, logger: Logger) => {
   ensureToken(options, logger);
   const multiMode = options.key.includes('*');
@@ -58,6 +75,7 @@ export const get = async (options: GetOptions, logger: Logger) => {
       limit: options.limit,
       key: options.key,
       collectionName: options.collectionName,
+      query: buildQuery(options),
     },
     logger
   );
@@ -178,6 +196,7 @@ export const remove = async (options: RemoveOptions, logger: Logger) => {
         token: options.token!,
         key: options.key,
         collectionName: options.collectionName,
+        query: buildQuery(options),
       },
       logger
     );
diff --git a/packages/cli/src/collections/request.ts b/packages/cli/src/collections/request.ts
index e48b5a815..ff9091222 100644
--- a/packages/cli/src/collections/request.ts
+++ b/packages/cli/src/collections/request.ts
@@ -17,6 +17,8 @@ type Options = {
   limit?: number;
 
   data?: any;
+
+  query?: any;
 };
 
 const DEFAULT_PAGE_SIZE = 1000;
@@ -39,10 +41,13 @@ export default async (
     Authorization: `Bearer ${options.token}`,
   };
 
-  const query: any = {
-    key: options.key,
-    limit: options.pageSize || DEFAULT_PAGE_SIZE,
-  };
+  const query: any = Object.assign(
+    {
+      key: options.key,
+      limit: options.pageSize || DEFAULT_PAGE_SIZE,
+    },
+    options.query
+  );
 
   const args: Partial<Dispatcher.RequestOptions> = {
     headers,

From 9731003787f20eba9cc74e5b36c77ccec5d16ad9 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Tue, 10 Dec 2024 15:28:45 +0000
Subject: [PATCH 18/23] cli: force collections key to be a string

---
 packages/cli/src/collections/command.ts | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 385d364ed..d7da99d75 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -78,8 +78,14 @@ const key = {
   name: 'key',
   yargs: {
     description: 'Key or key pattern to retrieve',
+    type: 'string',
     demand: true,
   },
+  ensure: (opts: Partial<CollectionsOptions>) => {
+    if (opts.key && typeof opts.key !== 'string') {
+      opts.key = `${opts.key}`;
+    }
+  },
 };
 
 // TODO this should default from env

From 16e8ed6c295989d4b27062ff40330198b7d78a9e Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 11 Dec 2024 14:51:35 +0000
Subject: [PATCH 19/23] cli: fix typing

---
 packages/cli/src/collections/command.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index d7da99d75..563cdbd8b 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -241,9 +241,7 @@ const items = {
 
 const setOptions = [
   collectionName,
-  // TODO in set, key does not support patterns
-  // We should document and catch this case
-  override(key, {
+  override(key as any, {
     demand: false,
   }),
   token,

From 7a3fa419ea36276313e942565e320b00c0c6d318 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 11 Dec 2024 15:37:20 +0000
Subject: [PATCH 20/23] worker: fix a typo in logging

---
 packages/ws-worker/src/server.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/ws-worker/src/server.ts b/packages/ws-worker/src/server.ts
index b0b925ea1..1ecabc108 100644
--- a/packages/ws-worker/src/server.ts
+++ b/packages/ws-worker/src/server.ts
@@ -155,7 +155,7 @@ async function setupCollections(options: ServerOptions, logger: Logger) {
       'WARNING: no collections URL provided. Collections service will not be enabled.'
     );
     logger.warn(
-      'Pass --collections-version or set WORKER_COLLECTIONS_URL to set the url'
+      'Pass --collections-url or set WORKER_COLLECTIONS_URL to set the url'
     );
     return;
   }

From 03f5b40656168bbdaff7daa8ced8117cda3e6fbb Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 11 Dec 2024 15:40:23 +0000
Subject: [PATCH 21/23] cli: refactor REPO_DIR warning message

---
 .changeset/lucky-pandas-explain.md      |  5 +++++
 packages/cli/src/collections/command.ts |  2 --
 packages/cli/src/collections/handler.ts |  1 -
 packages/cli/src/commands.ts            | 19 -------------------
 packages/cli/src/options.ts             | 12 ++++++++++++
 5 files changed, 17 insertions(+), 22 deletions(-)
 create mode 100644 .changeset/lucky-pandas-explain.md

diff --git a/.changeset/lucky-pandas-explain.md b/.changeset/lucky-pandas-explain.md
new file mode 100644
index 000000000..3c01b1e54
--- /dev/null
+++ b/.changeset/lucky-pandas-explain.md
@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Adjust OPENFN_REPO_DIR warning message
diff --git a/packages/cli/src/collections/command.ts b/packages/cli/src/collections/command.ts
index 563cdbd8b..5ad31b8ca 100644
--- a/packages/cli/src/collections/command.ts
+++ b/packages/cli/src/collections/command.ts
@@ -88,8 +88,6 @@ const key = {
   },
 };
 
-// TODO this should default from env
-// TODO this is used by other args
 const token = {
   name: 'pat',
   yargs: {
diff --git a/packages/cli/src/collections/handler.ts b/packages/cli/src/collections/handler.ts
index 3e421d289..7fbdebfec 100644
--- a/packages/cli/src/collections/handler.ts
+++ b/packages/cli/src/collections/handler.ts
@@ -130,7 +130,6 @@ export const set = async (options: SetOptions, logger: Logger) => {
   } else if (options.key && options.value) {
     const resolvedPath = path.resolve(options.value);
     logger.debug('Loading value from ', resolvedPath);
-    // TODO throw if key contains a *
 
     // set a single item
     const data = await readFile(path.resolve(options.value), 'utf8');
diff --git a/packages/cli/src/commands.ts b/packages/cli/src/commands.ts
index 29127bb59..048803719 100644
--- a/packages/cli/src/commands.ts
+++ b/packages/cli/src/commands.ts
@@ -89,25 +89,6 @@ const parse = async (options: Opts, log?: Logger) => {
     ) as string[];
   }
 
-  // TODO Please fix this joe!
-  // Put the validation inside the repoDir option
-
-  // TODO it would be nice to do this in the repoDir option, but
-  // the logger isn't available yet
-  if (
-    !/^(pull|deploy|test|version|apollo|collections-get|collections-set)$/.test(
-      options.command!
-    ) &&
-    !options.repoDir
-  ) {
-    logger.warn(
-      'WARNING: no repo module dir found! Using the default (/tmp/repo)'
-    );
-    logger.warn(
-      'You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI'
-    );
-  }
-
   const handler = handlers[options.command!];
 
   if (!handler) {
diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts
index 3082082b8..9b2955b67 100644
--- a/packages/cli/src/options.ts
+++ b/packages/cli/src/options.ts
@@ -368,6 +368,18 @@ export const repoDir: CLIOption = {
     description: 'Provide a path to the repo root dir',
     default: process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
   }),
+  ensure: (opts) => {
+    if (opts.repoDir === DEFAULT_REPO_DIR) {
+      // Note that we don't use the logger here - it's not been created yet
+      console.warn(
+        'WARNING: no repo module dir found! Using the default (/tmp/repo)'
+      );
+      console.warn(
+        'You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI'
+      );
+      console.log();
+    }
+  },
 };
 
 export const start: CLIOption = {

From 3a95d3b09fcfdec2b978654996169582e0cce803 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 11 Dec 2024 15:41:08 +0000
Subject: [PATCH 22/23] changeset

---
 .changeset/four-pigs-pull.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 .changeset/four-pigs-pull.md

diff --git a/.changeset/four-pigs-pull.md b/.changeset/four-pigs-pull.md
new file mode 100644
index 000000000..3f0f3283c
--- /dev/null
+++ b/.changeset/four-pigs-pull.md
@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Add collections command

From e5acbafc3504de5b2b0977922e28d4e914c7ea13 Mon Sep 17 00:00:00 2001
From: Joe Clark <jclark@openfn.org>
Date: Wed, 11 Dec 2024 15:49:39 +0000
Subject: [PATCH 23/23] version: cli@1.9.0

---
 .changeset/four-pigs-pull.md       |  5 -----
 .changeset/lucky-pandas-explain.md |  5 -----
 packages/cli/CHANGELOG.md          | 10 ++++++++++
 packages/cli/package.json          |  2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)
 delete mode 100644 .changeset/four-pigs-pull.md
 delete mode 100644 .changeset/lucky-pandas-explain.md

diff --git a/.changeset/four-pigs-pull.md b/.changeset/four-pigs-pull.md
deleted file mode 100644
index 3f0f3283c..000000000
--- a/.changeset/four-pigs-pull.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@openfn/cli': minor
----
-
-Add collections command
diff --git a/.changeset/lucky-pandas-explain.md b/.changeset/lucky-pandas-explain.md
deleted file mode 100644
index 3c01b1e54..000000000
--- a/.changeset/lucky-pandas-explain.md
+++ /dev/null
@@ -1,5 +0,0 @@
----
-'@openfn/cli': patch
----
-
-Adjust OPENFN_REPO_DIR warning message
diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md
index d92e20c1a..59cc73aa5 100644
--- a/packages/cli/CHANGELOG.md
+++ b/packages/cli/CHANGELOG.md
@@ -1,5 +1,15 @@
 # @openfn/cli
 
+## 1.9.0
+
+### Minor Changes
+
+- 3a95d3b: Add collections command
+
+### Patch Changes
+
+- 03f5b40: Adjust OPENFN_REPO_DIR warning message
+
 ## 1.8.11
 
 ### Patch Changes
diff --git a/packages/cli/package.json b/packages/cli/package.json
index 8bcefe462..f80f5ffea 100644
--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.8.11",
+  "version": "1.9.0",
   "description": "CLI devtools for the openfn toolchain.",
   "engines": {
     "node": ">=18",