From 28831f6ff4f1455b066087e83c166a1acf49af48 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 27 Feb 2024 14:02:33 +0300 Subject: [PATCH 1/6] update query function add options and callback params. --- packages/salesforce/ast.json | 38 ++++++++++++-- packages/salesforce/src/Adaptor.js | 79 ++++++++++++++++++++++++------ 2 files changed, 99 insertions(+), 18 deletions(-) diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json index 3278faeb0..6520a0d82 100644 --- a/packages/salesforce/ast.json +++ b/packages/salesforce/ast.json @@ -204,7 +204,9 @@ { "name": "query", "params": [ - "qs" + "qs", + "options", + "callback" ], "docs": { "description": "Execute an SOQL query.\nNote that in an event of a query error,\nerror logs will be printed but the operation will not throw the error.", @@ -216,7 +218,7 @@ }, { "title": "example", - "description": "query(`SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);" + "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);" }, { "title": "function", @@ -232,6 +234,36 @@ }, "name": "qs" }, + { + "title": "param", + "description": "Options passed to the bulk api.", + "type": { + "type": "NameExpression", + "name": "Object" + }, + "name": "options" + }, + { + "title": "param", + "description": "Fetch next records if available.", + "type": { + "type": "OptionalType", + "expression": { + "type": "NameExpression", + "name": "boolean" + } + }, + "name": "options.autoFetch" + }, + { + "title": "param", + "description": "A callback to execute once the record is retrieved", + "type": { + "type": "NameExpression", + "name": "Function" + }, + "name": "callback" + }, { "title": "returns", "description": null, @@ -242,7 +274,7 @@ } ] }, - "valid": true + "valid": false }, { "name": "bulkQuery", diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js index f23c0445c..b92eef24b 100644 --- a/packages/salesforce/src/Adaptor.js +++ b/packages/salesforce/src/Adaptor.js @@ -150,33 +150,82 @@ export function retrieve(sObject, id, callback) { * error logs will be printed but the operation will not throw the error. * @public * @example - * query(`SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`); + * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`); + * @example Query more records if next records are available + * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true }); * @function * @param {String} qs - A query string. + * @param {Object} options - Options passed to the bulk api. + * @param {boolean} [options.autoFetch] - Fetch next records if available. + * @param {Function} callback - A callback to execute once the record is retrieved * @returns {Operation} */ -export function query(qs) { - return state => { +export function query(qs, options, callback = s => s) { + return async state => { + let done = false; + let qResult = null; + let result = []; + const { connection } = state; - const resolvedQs = expandReferences(qs)(state); + const [resolvedQs, resolvedOptions] = newExpandReferences( + state, + qs, + options + ); + const { autoFetch } = { ...{ autoFetch: false }, ...resolvedOptions }; + console.log(`Executing query: ${resolvedQs}`); + try { + qResult = await connection.query(resolvedQs); + } catch (err) { + const { message, errorCode } = err; + console.log(`Error ${errorCode}: ${message}`); + throw err; + } - return connection.query(resolvedQs, function (err, result) { - if (err) { - const { message, errorCode } = err; - console.log(`Error ${errorCode}: ${message}`); - throw err; + if (qResult.totalSize > 0) { + console.log('Total records', qResult.totalSize); + + while (!done) { + result.push(qResult); + + if (qResult.done) { + done = true; + } else if (autoFetch) { + console.log( + 'Fetched records so far', + result.map(ref => ref.records).flat().length + ); + console.log('Fetching next records...'); + try { + qResult = await connection.request({ url: qResult.nextRecordsUrl }); + } catch (err) { + const { message, errorCode } = err; + console.log(`Error ${errorCode}: ${message}`); + throw err; + } + } else { + done = true; + } } console.log( - 'Results retrieved and pushed to position [0] of the references array.' + 'Done ✔ retrieved records', + result.map(ref => ref.records).flat().length ); + } else { + console.log('No records found.'); + } - return { - ...state, - references: [result, ...state.references], - }; - }); + console.log( + 'Results retrieved and pushed to position [0] of the references array.' + ); + + const nextState = { + ...state, + references: [result, ...state.references], + }; + return callback(nextState); }; } From 70ee6bfba69b13822500bd4d99b1e38ebdff793b Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 27 Feb 2024 14:03:25 +0300 Subject: [PATCH 2/6] add callback support in parseXML --- packages/http/src/Adaptor.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/http/src/Adaptor.js b/packages/http/src/Adaptor.js index a0d5f4473..d15d3defe 100644 --- a/packages/http/src/Adaptor.js +++ b/packages/http/src/Adaptor.js @@ -167,9 +167,10 @@ export function del(path, params, callback) { * @function * @param {String} body - data string to be parsed * @param {function} script - script for extracting data + * @param {function} callback - (Optional) Callback function * @returns {Operation} */ -export function parseXML(body, script) { +export function parseXML(body, script, callback = s => s) { return state => { const resolvedBody = expandReferences(body)(state); const $ = cheerio.load(resolvedBody); @@ -179,12 +180,12 @@ export function parseXML(body, script) { const result = script($); try { const r = JSON.parse(result); - return composeNextState(state, r); + return callback(composeNextState(state, r)); } catch (e) { - return composeNextState(state, { body: result }); + return callback(composeNextState(state, { body: result })); } } else { - return composeNextState(state, { body: resolvedBody }); + return callback(composeNextState(state, { body: resolvedBody })); } }; } From 49b525ed84057520a2f6d80e6e56a672f5af8ae1 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 27 Feb 2024 14:35:01 +0300 Subject: [PATCH 3/6] build salesforce ast --- packages/salesforce/ast.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json index 6520a0d82..1c2f92e5b 100644 --- a/packages/salesforce/ast.json +++ b/packages/salesforce/ast.json @@ -220,6 +220,11 @@ "title": "example", "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);" }, + { + "title": "example", + "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true });", + "caption": "Query more records if next records are available" + }, { "title": "function", "description": null, From 1d2fddf80c3d92bade17514d6909352eeacca340 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 27 Feb 2024 14:35:17 +0300 Subject: [PATCH 4/6] improve parseXML to use callback --- packages/http/ast.json | 19 ++++++++++++-- packages/http/src/Adaptor.js | 48 ++++++++++++++---------------------- packages/http/src/Utils.js | 23 +++++++++++++++++ 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/packages/http/ast.json b/packages/http/ast.json index 2f48e72b7..49d390560 100644 --- a/packages/http/ast.json +++ b/packages/http/ast.json @@ -392,7 +392,8 @@ "name": "parseXML", "params": [ "body", - "script" + "script", + "callback" ], "docs": { "description": "Parse XML with the Cheerio parser", @@ -404,7 +405,12 @@ }, { "title": "example", - "description": "parseXML(body, function($){\n return $(\"table[class=your_table]\").parsetable(true, true, true);\n })" + "description": "parseXML(\n (state) => state.response,\n ($) => {\n return $(\"table[class=your_table]\").parsetable(true, true, true);\n }\n);" + }, + { + "title": "example", + "description": "parseXML(\n (state) => state.response,\n ($) => {\n return $(\"table[class=your_table]\").parsetable(true, true, true);\n },\n (next) => ({ ...next, results: next.data.body })\n);", + "caption": "Using parseXML with a callback" }, { "title": "function", @@ -429,6 +435,15 @@ }, "name": "script" }, + { + "title": "param", + "description": "(Optional) Callback function", + "type": { + "type": "NameExpression", + "name": "function" + }, + "name": "callback" + }, { "title": "returns", "description": null, diff --git a/packages/http/src/Adaptor.js b/packages/http/src/Adaptor.js index d15d3defe..cdd342db9 100644 --- a/packages/http/src/Adaptor.js +++ b/packages/http/src/Adaptor.js @@ -1,12 +1,5 @@ -import { - execute as commonExecute, - composeNextState, - expandReferences, -} from '@openfn/language-common'; -import cheerio from 'cheerio'; -import cheerioTableparser from 'cheerio-tableparser'; - -import { request as sendRequest } from './Utils'; +import { execute as commonExecute } from '@openfn/language-common'; +import { request as sendRequest, xmlParser } from './Utils'; /** * Options provided to the HTTP request @@ -161,33 +154,28 @@ export function del(path, params, callback) { * Parse XML with the Cheerio parser * @public * @example - * parseXML(body, function($){ - * return $("table[class=your_table]").parsetable(true, true, true); - * }) + * parseXML( + * (state) => state.response, + * ($) => { + * return $("table[class=your_table]").parsetable(true, true, true); + * } + * ); + * @example Using parseXML with a callback + * parseXML( + * (state) => state.response, + * ($) => { + * return $("table[class=your_table]").parsetable(true, true, true); + * }, + * (next) => ({ ...next, results: next.data.body }) + * ); * @function * @param {String} body - data string to be parsed * @param {function} script - script for extracting data * @param {function} callback - (Optional) Callback function * @returns {Operation} */ -export function parseXML(body, script, callback = s => s) { - return state => { - const resolvedBody = expandReferences(body)(state); - const $ = cheerio.load(resolvedBody); - cheerioTableparser($); - - if (script) { - const result = script($); - try { - const r = JSON.parse(result); - return callback(composeNextState(state, r)); - } catch (e) { - return callback(composeNextState(state, { body: result })); - } - } else { - return callback(composeNextState(state, { body: resolvedBody })); - } - }; +export function parseXML(body, script, callback) { + return xmlParser(body, script, callback); } export { diff --git a/packages/http/src/Utils.js b/packages/http/src/Utils.js index 9aa90fae3..d19b0640e 100644 --- a/packages/http/src/Utils.js +++ b/packages/http/src/Utils.js @@ -6,6 +6,9 @@ import { makeBasicAuthHeader, } from '@openfn/language-common/util'; +import * as cheerio from 'cheerio'; +import cheerioTableparser from 'cheerio-tableparser'; + export function addBasicAuth(configuration = {}, headers) { const { username, password } = configuration; if (username && password) { @@ -84,3 +87,23 @@ export function request(method, path, params, callback = s => s) { }); }; } + +export function xmlParser(body, script, callback = s => s) { + return state => { + const [resolvedBody] = expandReferences(state, body); + const $ = cheerio.load(resolvedBody); + cheerioTableparser($); + + if (script) { + const result = script($); + try { + const r = JSON.parse(result); + return callback(composeNextState(state, r)); + } catch (e) { + return callback(composeNextState(state, { body: result })); + } + } else { + return callback(composeNextState(state, { body: resolvedBody })); + } + }; +} From cfe1ccbda7d8a21b0daf1b65aa854b430ed5320f Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 27 Feb 2024 14:37:22 +0300 Subject: [PATCH 5/6] add changeset --- .changeset/fresh-zoos-type.md | 5 +++++ .changeset/three-dragons-sleep.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/fresh-zoos-type.md create mode 100644 .changeset/three-dragons-sleep.md diff --git a/.changeset/fresh-zoos-type.md b/.changeset/fresh-zoos-type.md new file mode 100644 index 000000000..fc069e846 --- /dev/null +++ b/.changeset/fresh-zoos-type.md @@ -0,0 +1,5 @@ +--- +'@openfn/language-salesforce': minor +--- + +Add options and callback params in query function diff --git a/.changeset/three-dragons-sleep.md b/.changeset/three-dragons-sleep.md new file mode 100644 index 000000000..00ae4e5b6 --- /dev/null +++ b/.changeset/three-dragons-sleep.md @@ -0,0 +1,5 @@ +--- +'@openfn/language-http': minor +--- + +Add callback support for parseXML From 27fc2323655bd4668d0a46cb737ac98968cd8468 Mon Sep 17 00:00:00 2001 From: Emmanuel Evance Date: Tue, 19 Mar 2024 16:02:25 +0300 Subject: [PATCH 6/6] bump version and update changelog --- .changeset/fresh-zoos-type.md | 5 ----- .changeset/three-dragons-sleep.md | 5 ----- packages/http/CHANGELOG.md | 6 ++++++ packages/http/package.json | 2 +- packages/salesforce/CHANGELOG.md | 6 ++++++ packages/salesforce/package.json | 2 +- 6 files changed, 14 insertions(+), 12 deletions(-) delete mode 100644 .changeset/fresh-zoos-type.md delete mode 100644 .changeset/three-dragons-sleep.md diff --git a/.changeset/fresh-zoos-type.md b/.changeset/fresh-zoos-type.md deleted file mode 100644 index fc069e846..000000000 --- a/.changeset/fresh-zoos-type.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/language-salesforce': minor ---- - -Add options and callback params in query function diff --git a/.changeset/three-dragons-sleep.md b/.changeset/three-dragons-sleep.md deleted file mode 100644 index 00ae4e5b6..000000000 --- a/.changeset/three-dragons-sleep.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/language-http': minor ---- - -Add callback support for parseXML diff --git a/packages/http/CHANGELOG.md b/packages/http/CHANGELOG.md index 4ea9edbd1..8e17a7460 100644 --- a/packages/http/CHANGELOG.md +++ b/packages/http/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/language-http +## 6.1.0 + +### Minor Changes + +- cfe1ccb: Add callback support for parseXML + ## 6.0.0 ### Major Changes diff --git a/packages/http/package.json b/packages/http/package.json index c7cb33f12..1698d4589 100644 --- a/packages/http/package.json +++ b/packages/http/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/language-http", - "version": "6.0.0", + "version": "6.1.0", "description": "An HTTP request language package for use with Open Function", "homepage": "https://docs.openfn.org", "repository": { diff --git a/packages/salesforce/CHANGELOG.md b/packages/salesforce/CHANGELOG.md index 9ea801cc2..3d2fce758 100644 --- a/packages/salesforce/CHANGELOG.md +++ b/packages/salesforce/CHANGELOG.md @@ -1,5 +1,11 @@ # @openfn/language-salesforce +## 4.6.0 + +### Minor Changes + +- cfe1ccb: Add options and callback params in query function + ## 4.5.2 ### Patch Changes diff --git a/packages/salesforce/package.json b/packages/salesforce/package.json index 83540ffe7..b7d006f4e 100644 --- a/packages/salesforce/package.json +++ b/packages/salesforce/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/language-salesforce", - "version": "4.5.2", + "version": "4.6.0", "description": "Salesforce Language Pack for OpenFn", "homepage": "https://docs.openfn.org", "exports": {