[Epic] New HTTP Adaptor & Utils (#457)

* wip add request util * wip: using undici client add tests * wip: add mock agent * wip: more tests * wip: test options * update undici * add changeset * fix typo * improve errorMap * fix undici * remove method in assertOK * improve signature * improve assertOK add tests for baseUrl option and errorMap * throw an error if response code match errorMap * remove duplicate property value * test if body is string and add default body timeout * wip: improvements in util * remove util.js * restructure util files * add recursive option * common: restructure build output * remove comments * update isValidHttpUrl minor improvements * rename a test * wip: parseAs * wip: update mockClient func * wip: refactor test * improve tests * better url parsing * remove comment * update response body * update content type check * remove comments * remove headers * common: adjust import * common: tweak request docs * wip: refactor * wip: refactor test * wip: post tests * wip: skip failed tests * wip: tests * common: updated unit tests * common: fix test * add request helper function * wip: clean up tests * add support for successcode in errorMap * fix broken test * remove undici skip test * remove cookies handling * remove unused packages * improve aut parsing * wip: followRedirect * wip: follow redirect * wip: form-data * http: update formdata tests * http: remove rediect test * improve logging * clean up unused packages and add http-status-codes * clean up deadcode * improve form-data test * add changeset * remove common changeset * improvements * add duration * add method and url to response body * add support for tls * improve jsdoc * update changelog * add requestBodyType function This function assert that body type and convert a plan json to a string * remove JSON.stringify * add unit test for requestBodyType * add helpers for http server * update dependencies * remove http helpers * add intergration test * add responseUrl function This function will return history url if found or request url * fix test * add koa and koa-sslify * add certificates for testing * update tests * move server into integration file * add server logic * update https port it turns out in ubuntu any port < 1024 needs root permission * use nodes https module remove koa implementation * http: move request helper into utils * common: typo in docs * http: fix headers and start updating tests * package lock * test updates * common: rename http helper response touse statusCode and statusMessage. Also remove the response url. * http: update tests * http: update tests * common: restore url to the response object A version of it anyway * http: refactor response log * common: update http error message * http: update to use new error handling * http: tidy ups * http: docs * http,common:docs * http: undocument maxRedirects Not willing to commit to it tbh * http: light refactor * common: another light refactor * lock node version in ci * common: restore default error map * http: fix form processing * http: remove file extension from import * common: deprecate old http functions Shoud print a warning in dev mode, but not in prod (platform or lightning) * http: expose a general request operation * common: update release notes * bump versions * lockfile * Update auto-generated documentation. * docs: improve markdown templates * tools: update docs * Update http docs * http: update request example * remove build artefacts * http,common: move some http stuff into common utils * Update template * template: update to use new helpers * template: remove unused code * readme * Update template Template should be locked at 1.0.0 and never published * common: don't print output if you're given a bad response * common: better url building * http: remove custom tags from docs * common: tweak URL parsing rules * common: better url reporting in errors * common: fix url parsing whoops * common: don't use URL.parse * add linting for undeclared variables Fixes issues in azure-storage, common, http, magpi, mailchimp, mongodb, resourcemap and twilio * bump versions * common: better handling of error responses in http * http: update changelog * http: changelog fixes --------- Co-authored-by: Emmanuel Evance <[email protected]>
OpenFn · Jan 24, 2024 · 56ae0dd · 56ae0dd
1 parent a1754a1
commit 56ae0dd
Show file tree

Hide file tree

Showing 90 changed files with 3,469 additions and 2,216 deletions.
diff --git a/.changeset/config.json b/.changeset/config.json
@@ -7,5 +7,5 @@
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": []
+  "ignore": ["@openfn/language-template"]
 }
diff --git a/.eslintrc.json b/.eslintrc.json
@@ -8,9 +8,17 @@
   },
   "rules": {
     "no-unused-vars": "warn",
-    "no-param-reassign": "error"
+    "no-param-reassign": "error",
+    "no-undef": "error"
   },
   "env": {
     "node": true
+  },
+  "globals": {
+    "FormData": "readonly",
+    "Map": "readonly",
+    "Symbol": "readonly",
+    "ReadableStream": "readonly",
+    "Promise": "readonly"
   }
 }
diff --git a/README.md b/README.md
@@ -133,23 +133,22 @@ export function yourFunctionName(arguments) {
 ```
 
 The logic is where you will prepare your API request. Build the URL, passing in
-any arguments if needed. Next, return the fetch request, making sure you set the
-Authorization, passing in any secrets from `state.configuration`.
+any arguments if needed. Then, using a helper function in common, return the
+fetch request, making sure you set the Authorization, passing in any secrets
+from `state.configuration`.
 
 ```js
+import { get } from '@openfn/language-common/util';
+
 export function getTodaysWeather(latitude = 25.52, longitude = 13.41) {
   return state => {
     const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&current_weather=true&hourly=temperature_2m,relativehumidity_2m,windspeed_10m`;
 
-    return (
-      fetch(url, {
-        headers: {
-          Authorization: `${state.configuration.username}:${state.configuration.password}`,
-        },
-      })
-        .then(response => response.json())
-        .then(data => ({ ...state, data }));
-    );
+    const headers = {
+      Authorization: `${state.configuration.username}:${state.configuration.password}`,
+    };
+    const result = await get(url, { headers });
+    return { ...state, data: result.body };
   };
 }
 ```
@@ -185,7 +184,7 @@ export function getTodaysWeather(latitude = 25.52, longitude = 13.41) {
 
 Import your newly defined function
 
-```
+```js
 import { functionName } from '../src/Adaptor.js';
 ```
 
@@ -194,7 +193,7 @@ to see how state should be passed to it).
 
 Make an assertion to check the result.
 
-```
+```js
 it('should xyz', async () => {
 const state = {
   configuration: {},
@@ -236,6 +235,9 @@ Look in the `.changesets` folder to see your change.
 
 Commit the changeset to the repo when you're ready.
 
+Note that the`template` adaptor should ideally never have its version
+increased - it should be locked at `1.0.0`.
+
 ## Releases
 
 New releases will be published to npm automatically when merging into the `main`
@@ -276,6 +278,25 @@ Examples:
 pnpm -C packages/salesforce build --watch
 ```
 
+### Docs
+
+Docs are generated from the JSDoc annotations in adaptors. They are output as
+markdown files in the `./docs` directly and not checked in to source control.
+
+The markdown output can be customised by overriding the built-in handlebars
+templates in jsoc2md.
+
+- Find the template you want to customise in [j2sdoc2md source()
+  https://github.com/jsdoc2md/dmd/tree/master/partials) (this can be tricky)
+- Copy the template contents
+- Paste into a file with the same name (this is important) in
+  `tools/build/src/partials`
+- Edit `tools/build/src/commands/docs.ts` and add the path to your new template
+  to jsdoc2md's `renderOpts` (see how the other .hbs files are loaded in)
+- Make your changes
+- Run `pnpm build docs` from root (or just one adaptor folder) and inspect the
+  generated `docs/index/md` file.
+
 ## Metadata
 
 Check the Wiki for the metadata creation guide:

diff --git a/packages/asana/package.json b/packages/asana/package.json
@@ -31,7 +31,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "^1.8.1"
+    "@openfn/language-common": "^1.12.0"
   },
   "devDependencies": {
     "@openfn/buildtools": "workspace:^1.0.2",

diff --git a/packages/azure-storage/CHANGELOG.md b/packages/azure-storage/CHANGELOG.md
@@ -1,5 +1,11 @@
 # @openfn/language-azure-storage
 
+## 1.0.1
+
+### Patch Changes
+
+- 6afba70: Add proper variable declaration (linting)
+
 ## 1.0.0
 
 ### Major Changes

diff --git a/packages/azure-storage/package.json b/packages/azure-storage/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-azure-storage",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "OpenFn adaptor for Azure Storage",
   "type": "module",
   "exports": {
@@ -27,7 +27,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "^1.11.1",
+    "@openfn/language-common": "^1.12.0",
     "@azure/storage-blob": "^12.15.0"
   },
   "devDependencies": {

diff --git a/packages/azure-storage/src/Adaptor.js b/packages/azure-storage/src/Adaptor.js
@@ -134,7 +134,7 @@ export function uploadBlob(blobName, content, uploadOptions, options = {}) {
     console.debug(
       `Content: '${finalContent}', length: '${finalContent.length}'`
     );
-    response = await blockBlobClient.upload(
+    const response = await blockBlobClient.upload(
       finalContent,
       finalContent.length,
       resolvedUploadOptions

diff --git a/packages/beyonic/package.json b/packages/beyonic/package.json
@@ -31,7 +31,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "^1.8.1",
+    "@openfn/language-common": "^1.12.0",
     "JSONPath": "^0.10.0",
     "lodash-fp": "^0.10.2",
     "superagent": "^8.0.0"

diff --git a/packages/bigquery/package.json b/packages/bigquery/package.json
@@ -21,7 +21,7 @@
   ],
   "dependencies": {
     "@google-cloud/bigquery": "^5.12.0",
-    "@openfn/language-common": "workspace:1.11.1",
+    "@openfn/language-common": "workspace:1.12.0",
     "csv-parse": "^4.16.3",
     "import": "0.0.6",
     "json2csv": "^5.0.7",

diff --git a/packages/cartodb/package.json b/packages/cartodb/package.json
@@ -22,7 +22,7 @@
   "dependencies": {
     "JSONPath": "^0.10.0",
     "json-sql": "^0.3.8",
-    "@openfn/language-common": "^1.8.1",
+    "@openfn/language-common": "^1.12.0",
     "lodash-fp": "^0.10.2",
     "superagent": "^3.7.0"
   },

diff --git a/packages/commcare/package.json b/packages/commcare/package.json
@@ -25,8 +25,8 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.11.1",
-    "@openfn/language-http": "workspace:^5.0.4",
+    "@openfn/language-common": "workspace:1.12.0",
+    "@openfn/language-http": "workspace:^6.0.0",
     "JSONPath": "^0.10.0",
     "form-data": "^4.0.0",
     "js2xmlparser": "^1.0.0",

diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md
@@ -1,5 +1,51 @@
 v0.4.0
 
+## 1.12.0
+
+### Minor Changes
+
+- 7f52699: New HTTP helper functions have been added to common in
+  `src/util/http.js`
+
+  These are based on the `undici` library. They are functions, not operations,
+  so they do not get and return state, and do not expand references.
+
+  They are designed to be used by other adaptors to make HTTP requests easier.
+
+  ## Usage
+
+  ```
+  // Import the helper function
+  import { get } from '@openfn/language-common/util'
+
+  // This is an example operation
+  export function get(id, callback) {
+    return async (state) => {
+      const [resolvedId] = expandReferences(
+        state,
+        id,
+      );
+
+      // Call the new common helper to fetch some json
+      const response = await get(`www.example.com/resource/{$resolvedId}`, { parseAs: 'json' });
+
+      // Return the response body as data, and also include the response object as a convenience
+      return {
+        ...state,
+        response,
+        data: response.body
+      }
+    }
+  }
+  ```
+
+  See the http adaptor for a reference implementation.
+
+  ## Deprecation notice
+
+  The existing http operations in `src/http.js` have been deprecated, and
+  adaptors should migrate to the new helpers.
+
 ## 1.11.1
 
 ### Patch Changes

diff --git a/packages/common/build.config.js b/packages/common/build.config.js
@@ -1,4 +1,8 @@
 // override build config with an extra entry point
 export default path => ({
-  entry: [`${path}/src/index.js`, `${path}/src/metadata.js`, `${path}/src/util.js`],
+  entry: {
+    index: `${path}/src/index.js`,
+    metadata: `${path}/src/metadata.js`,
+    util: `${path}/src/util/index.js`,
+  },
 });
diff --git a/packages/common/package.json b/packages/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-common",
-  "version": "1.11.1",
+  "version": "1.12.0",
   "description": "Common Expressions for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -9,8 +9,8 @@
   },
   "scripts": {
     "build": "pnpm clean && build-adaptor common",
-    "test": "mocha --experimental-specifier-resolution=node --no-warnings",
-    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings",
+    "test": "mocha --experimental-specifier-resolution=node --no-warnings --recursive",
+    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings --recursive",
     "clean": "rimraf dist types docs",
     "pack": "pnpm pack --pack-destination ../../dist",
     "lint": "eslint src"
@@ -46,9 +46,10 @@
     "csv-parse": "^5.4.0",
     "csvtojson": "^2.0.10",
     "date-fns": "^2.25.0",
+    "http-status-codes": "^2.3.0",
     "jsonpath-plus": "^4.0.0",
     "lodash": "^4.17.19",
-    "undici": "^5.22.1"
+    "undici": "^5.23.0"
   },
   "devDependencies": {
     "@openfn/buildtools": "workspace:^1.0.2",

diff --git a/packages/common/src/Adaptor.js b/packages/common/src/Adaptor.js
@@ -10,7 +10,7 @@ import { request } from 'undici';
 import { expandReferences as newExpandReferences } from './util';
 
 export * as beta from './beta';
-export * as http from './http';
+export * as http from './http.deprecated';
 export * as dateFns from './dateFns';
 
 const schemaCache = {};

diff --git a/packages/common/src/http.js → packages/common/src/http.deprecated.js b/packages/common/src/http.js → packages/common/src/http.deprecated.js
@@ -1,9 +1,27 @@
+/**
+ * === DEPRECATION NOTICE ===
+ *
+ * These common HTTP utils have been deprecated in favour of the new functions in util/http.js
+ *
+ * To ease migration onto the new APIs, these functions will be left in place and will continue to work.
+ *
+ * ===========================
+ */
+
 import { expandReferences, splitKeys } from './Adaptor';
 import axios from 'axios';
 import https from 'https';
 
 export { axios };
 
+const printDeprecationNotice = name => {
+  if (process.env.NODE_ENV !== 'production') {
+    console.warn(
+      `The common.http.¬${name} function has been deprecated. This adaptor should migrate to use common.util.http instead.`
+    );
+  }
+};
+
 /**
  * Recursively resolves objects that have resolvable values (functions), but
  * omits HTTP request specific modules like `FormData`.
@@ -67,6 +85,7 @@ function withAgent(params) {
  */
 export function get(requestParams) {
   return state => {
+    printDeprecationNotice('get');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'get', ...withAgent(params) });
@@ -96,6 +115,7 @@ export function get(requestParams) {
  */
 export function post(requestParams) {
   return state => {
+    printDeprecationNotice('post');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'post', ...withAgent(params) });
@@ -115,6 +135,7 @@ export function post(requestParams) {
  */
 function del(requestParams) {
   return state => {
+    printDeprecationNotice('del');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'delete', ...withAgent(params) });
@@ -136,6 +157,7 @@ export { del as delete };
  */
 export function head(requestParams) {
   return state => {
+    printDeprecationNotice('head');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'head', ...withAgent(params) });
@@ -156,6 +178,7 @@ export function head(requestParams) {
  */
 export function put(requestParams) {
   return state => {
+    printDeprecationNotice('put');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'put', ...withAgent(params) });
@@ -176,6 +199,7 @@ export function put(requestParams) {
  */
 export function patch(requestParams) {
   return state => {
+    printDeprecationNotice('patch');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'patch', ...withAgent(params) });
@@ -195,6 +219,7 @@ export function patch(requestParams) {
  */
 export function options(requestParams) {
   return state => {
+    printDeprecationNotice('options');
     const params = expandRequestReferences(requestParams)(state);
 
     return axios({ method: 'options', ...withAgent(params) });

diff --git a/packages/common/src/metadata.js b/packages/common/src/metadata.js
@@ -22,7 +22,7 @@ export const createEntity = (name, type, props = {}, children) => {
         this.children = name ? {} : [];
       }
       if (name) {
-        if (isArray(this.children)) {
+        if (Array.isArray(this.children)) {
           throw new Error('Cannot add a named entity to child array');
         }
         this.children[name] = e;