From d71915f48bab5e03e2e2577af1c05235802afa02 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Wed, 17 Jul 2024 13:49:20 +0300
Subject: [PATCH 01/22] remove axios

---
 packages/salesforce/package.json | 1 -
 pnpm-lock.yaml                   | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/packages/salesforce/package.json b/packages/salesforce/package.json
index dff6475c5..78ce5179d 100644
--- a/packages/salesforce/package.json
+++ b/packages/salesforce/package.json
@@ -33,7 +33,6 @@
   "dependencies": {
     "@openfn/language-common": "workspace:*",
     "any-ascii": "^0.3.2",
-    "axios": "^0.21.4",
     "jsforce": "^1.11.1",
     "lodash": "^4.17.21"
   },
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 3d77540fe..491dbe9cb 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1624,9 +1624,6 @@ importers:
       any-ascii:
         specifier: ^0.3.2
         version: 0.3.2
-      axios:
-        specifier: ^0.21.4
-        version: 0.21.4
       jsforce:
         specifier: ^1.11.1
         version: 1.11.1

From fd7be42083ffda07da1adb129171579e7e5caa19 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Wed, 17 Jul 2024 13:49:40 +0300
Subject: [PATCH 02/22] move helper functions to Utils

---
 packages/salesforce/src/Utils.js | 136 +++++++++++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100644 packages/salesforce/src/Utils.js

diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
new file mode 100644
index 000000000..62e47e85b
--- /dev/null
+++ b/packages/salesforce/src/Utils.js
@@ -0,0 +1,136 @@
+import jsforce from 'jsforce';
+
+function getConnection(state, options) {
+  const { apiVersion } = state.configuration;
+
+  const apiVersionRegex = /^\d{2}\.\d$/;
+
+  if (apiVersion && apiVersionRegex.test(apiVersion)) {
+    options.version = apiVersion;
+  } else {
+    options.version = '47.0';
+  }
+  console.log('Using Salesforce API version:', options.version);
+
+  return new jsforce.Connection(options);
+}
+
+async function createBasicAuthConnection(state) {
+  const { loginUrl, username, password, securityToken } = state.configuration;
+
+  const connection = getConnection(state, { loginUrl });
+
+  await connection
+    .login(username, securityToken ? password + securityToken : password)
+    .catch(e => {
+      console.error(`Failed to connect to salesforce as ${username}`);
+      throw e;
+    });
+
+  console.info(`Connected to salesforce as ${username}.`);
+
+  return {
+    ...state,
+    connection,
+  };
+}
+
+function createAccessTokenConnection(state) {
+  const { instance_url, access_token } = state.configuration;
+
+  const connection = getConnection(state, {
+    instanceUrl: instance_url,
+    accessToken: access_token,
+  });
+
+  console.log(`Connected with ${connection._sessionType} session type`);
+
+  return {
+    ...state,
+    connection,
+  };
+}
+
+/**
+ * Creates a connection to Salesforce using Basic Auth or OAuth.
+ * @function createConnection
+ * @private
+ * @param {State} state - Runtime state.
+ * @returns {State}
+ */
+export function createConnection(state) {
+  if (state.connection) {
+    return state;
+  }
+
+  const { access_token } = state.configuration;
+
+  return access_token
+    ? createAccessTokenConnection(state)
+    : createBasicAuthConnection(state);
+}
+
+/**
+ * Removes state.connection from state.
+ * @example
+ * removeConnection(state)
+ * @function
+ * @private
+ * @param {State} state
+ * @returns {State}
+ */
+export function removeConnection(state) {
+  delete state.connection;
+  return state;
+}
+
+export async function pollJobResult(conn, job, pollInterval, pollTimeout) {
+  let attempt = 0;
+
+  const maxPollingAttempts = Math.floor(pollTimeout / pollInterval);
+
+  while (attempt < maxPollingAttempts) {
+    // Make an HTTP GET request to check the job status
+    const jobInfo = await conn
+      .request({
+        method: 'GET',
+        url: `/services/data/v${conn.version}/jobs/query/${job.id}`,
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      })
+      .catch(error => {
+        console.log('Failed to fetch job information', error);
+      });
+
+    if (jobInfo && jobInfo.state === 'JobComplete') {
+      const response = await conn.request({
+        method: 'GET',
+        url: `/services/data/v${conn.version}/jobs/query/${job.id}/results`,
+        headers: {
+          'Content-Type': 'application/json',
+        },
+      });
+
+      console.log('Job result retrieved', response.length);
+      return response;
+    } else {
+      // Handle maxPollingAttempts
+      if (attempt + 1 === maxPollingAttempts) {
+        console.error(
+          'Maximum polling attempt reached, Please increase pollInterval and pollTimeout'
+        );
+        throw new Error(`Polling time out. Job Id = ${job.id}`);
+      }
+      console.log(
+        `Attempt ${attempt + 1} - Job ${jobInfo.id} is still in ${
+          jobInfo.state
+        }:`
+      );
+    }
+
+    // Wait for the polling interval before the next attempt
+    await new Promise(resolve => setTimeout(resolve, pollInterval));
+    attempt++;
+  }
+}

From 454b0eb81176c6e164d3c53d4d98f10df34635d3 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Wed, 17 Jul 2024 13:50:33 +0300
Subject: [PATCH 03/22] function cleanup

---
 packages/salesforce/ast.json             |  81 ---
 packages/salesforce/src/Adaptor.js       | 858 ++++++++---------------
 packages/salesforce/test/Adaptor.test.js | 104 +--
 3 files changed, 299 insertions(+), 744 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index 19f0db47c..ed54ae5ed 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -687,87 +687,6 @@
       },
       "valid": true
     },
-    {
-      "name": "createIf",
-      "params": [
-        "logical",
-        "sObject",
-        "attrs"
-      ],
-      "docs": {
-        "description": "Create a new sObject if conditions are met.\n\n**The `createIf()` function has been deprecated. Use `fnIf(condition,create())` instead.**",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "createIf(true, 'obj_name', {\n  attr1: \"foo\",\n  attr2: \"bar\"\n})"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
-          {
-            "title": "param",
-            "description": "a logical statement that will be evaluated.",
-            "type": {
-              "type": "NameExpression",
-              "name": "boolean"
-            },
-            "name": "logical"
-          },
-          {
-            "title": "param",
-            "description": "API name of the sObject.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "sObject"
-          },
-          {
-            "title": "param",
-            "description": "Field attributes for the new object.",
-            "type": {
-              "type": "UnionType",
-              "elements": [
-                {
-                  "type": "NameExpression",
-                  "name": "object"
-                },
-                {
-                  "type": "TypeApplication",
-                  "expression": {
-                    "type": "NameExpression",
-                    "name": "Array"
-                  },
-                  "applications": [
-                    {
-                      "type": "NameExpression",
-                      "name": "object"
-                    }
-                  ]
-                }
-              ]
-            },
-            "name": "attrs"
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "Operation"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
     {
       "name": "upsert",
       "params": [
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 77cf09a10..5457289fb 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -20,8 +20,8 @@ import {
 } from '@openfn/language-common';
 
 import { expandReferences as newExpandReferences } from '@openfn/language-common/util';
+import * as util from './Utils';
 
-import jsforce from 'jsforce';
 import flatten from 'lodash/flatten';
 
 let anyAscii = undefined;
@@ -35,328 +35,31 @@ const loadAnyAscii = state =>
   });
 
 /**
- * Adds a lookup relation or 'dome insert' to a record.
- * @public
- * @example
- * Data Sourced Value:
- *  relationship("relationship_name__r", "externalID on related object", dataSource("path"))
- * Fixed Value:
- *  relationship("relationship_name__r", "externalID on related object", "hello world")
- * @function
- * @param {string} relationshipName - `__r` relationship field on the record.
- * @param {string} externalId - Salesforce ExternalID field.
- * @param {string} dataSource - resolvable source.
- * @returns {object}
- */
-export function relationship(relationshipName, externalId, dataSource) {
-  return field(relationshipName, state => {
-    if (typeof dataSource == 'function') {
-      return { [externalId]: dataSource(state) };
-    }
-    return { [externalId]: dataSource };
-  });
-}
-
-/**
- * Prints the total number of all available sObjects and pushes the result to `state.references`.
- * @public
- * @example
- * describeAll()
- * @function
- * @returns {Operation}
- */
-export function describeAll() {
-  return state => {
-    const { connection } = state;
-
-    return connection.describeGlobal().then(result => {
-      const { sobjects } = result;
-      console.log(`Retrieved ${sobjects.length} sObjects`);
-
-      return {
-        ...state,
-        references: [sobjects, ...state.references],
-      };
-    });
-  };
-}
-
-/**
- * Prints an sObject metadata and pushes the result to state.references
- * @public
- * @example
- * describe('obj_name')
+ * Executes an operation.
  * @function
- * @param {string} sObject - API name of the sObject.
- * @returns {Operation}
+ * @param {Operation} operations - Operations
+ * @returns {State}
  */
-export function describe(sObject) {
-  return state => {
-    const { connection } = state;
-
-    const objectName = expandReferences(sObject)(state);
-
-    return connection
-      .sobject(objectName)
-      .describe()
-      .then(result => {
-        console.log('Label : ' + result.label);
-        console.log('Num of Fields : ' + result.fields.length);
-
-        return {
-          ...state,
-          references: [result, ...state.references],
-        };
-      });
+export function execute(...operations) {
+  const initialState = {
+    logger: {
+      info: console.info.bind(console),
+      debug: console.log.bind(console),
+    },
+    references: [],
+    data: null,
+    configuration: {},
   };
-}
 
-/**
- * Retrieves a Salesforce sObject(s).
- * @public
- * @example
- * retrieve('ContentVersion', '0684K0000020Au7QAE/VersionData');
- * @function
- * @param {string} sObject - The sObject to retrieve
- * @param {string} id - The id of the record
- * @param {function} callback - A callback to execute once the record is retrieved
- * @returns {Operation}
- */
-export function retrieve(sObject, id, callback) {
   return state => {
-    const { connection } = state;
-
-    const finalId = expandReferences(id)(state);
-
-    return connection
-      .sobject(sObject)
-      .retrieve(finalId)
-      .then(result => {
-        return {
-          ...state,
-          references: [result, ...state.references],
-        };
-      })
-      .then(state => {
-        if (callback) {
-          return callback(state);
-        }
-        return state;
-      });
-  };
-}
-
-/**
- * Execute an SOQL query.
- * Note that in an event of a query error,
- * error logs will be printed but the operation will not throw the error.
- *
- * The Salesforce query API is subject to rate limits, {@link https://sforce.co/3W9zyaQ See for more details}.
- * @public
- * @example
- * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);
- * @example <caption>Query more records if next records are available</caption>
- * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true });
- * @function
- * @param {string} qs - A query string. Must be less than `4000` characters in WHERE clause
- * @param {object} options - Options passed to the bulk api.
- * @param {boolean} [options.autoFetch=false] - Fetch next records if available.
- * @param {function} callback - A callback to execute once the record is retrieved
- * @returns {Operation}
- */
-export function query(qs, options, callback = s => s) {
-  return async state => {
-    let done = false;
-    let qResult = null;
-    let result = [];
-
-    const { connection } = state;
-    const [resolvedQs, resolvedOptions] = newExpandReferences(
-      state,
-      qs,
-      options
-    );
-    const { autoFetch } = { ...{ autoFetch: false }, ...resolvedOptions };
-
-    console.log(`Executing query: ${resolvedQs}`);
-    try {
-      qResult = await connection.query(resolvedQs);
-    } catch (err) {
-      const { message, errorCode } = err;
-      console.log(`Error ${errorCode}: ${message}`);
-      throw err;
-    }
-
-    if (qResult.totalSize > 0) {
-      console.log('Total records', qResult.totalSize);
-
-      while (!done) {
-        result.push(qResult);
-
-        if (qResult.done) {
-          done = true;
-        } else if (autoFetch) {
-          console.log(
-            'Fetched records so far',
-            result.map(ref => ref.records).flat().length
-          );
-          console.log('Fetching next records...');
-          try {
-            qResult = await connection.request({ url: qResult.nextRecordsUrl });
-          } catch (err) {
-            const { message, errorCode } = err;
-            console.log(`Error ${errorCode}: ${message}`);
-            throw err;
-          }
-        } else {
-          done = true;
-        }
-      }
-
-      console.log(
-        'Done ✔ retrieved records',
-        result.map(ref => ref.records).flat().length
-      );
-    } else {
-      result.push(qResult);
-      console.log('No records found.');
-    }
-
-    console.log(
-      'Results retrieved and pushed to position [0] of the references array.'
-    );
-
-    const nextState = {
-      ...state,
-      references: [result, ...state.references],
-    };
-    return callback(nextState);
-  };
-}
-
-async function pollJobResult(conn, job, pollInterval, pollTimeout) {
-  let attempt = 0;
-
-  const maxPollingAttempts = Math.floor(pollTimeout / pollInterval);
-
-  while (attempt < maxPollingAttempts) {
-    // Make an HTTP GET request to check the job status
-    const jobInfo = await conn
-      .request({
-        method: 'GET',
-        url: `/services/data/v${conn.version}/jobs/query/${job.id}`,
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      })
-      .catch(error => {
-        console.log('Failed to fetch job information', error);
-      });
-
-    if (jobInfo && jobInfo.state === 'JobComplete') {
-      const response = await conn.request({
-        method: 'GET',
-        url: `/services/data/v${conn.version}/jobs/query/${job.id}/results`,
-        headers: {
-          'Content-Type': 'application/json',
-        },
-      });
-
-      console.log('Job result retrieved', response.length);
-      return response;
-    } else {
-      // Handle maxPollingAttempts
-      if (attempt + 1 === maxPollingAttempts) {
-        console.error(
-          'Maximum polling attempt reached, Please increase pollInterval and pollTimeout'
-        );
-        throw new Error(`Polling time out. Job Id = ${job.id}`);
-      }
-      console.log(
-        `Attempt ${attempt + 1} - Job ${jobInfo.id} is still in ${
-          jobInfo.state
-        }:`
-      );
-    }
-
-    // Wait for the polling interval before the next attempt
-    await new Promise(resolve => setTimeout(resolve, pollInterval));
-    attempt++;
-  }
-}
-
-const defaultOptions = {
-  pollTimeout: 90000, // in ms
-  pollInterval: 3000, // in ms
-};
-/**
- * Execute an SOQL Bulk Query.
- * This function uses bulk query to efficiently query large data sets and reduce the number of API requests.
- * `bulkQuery()` uses {@link https://sforce.co/4azgczz Bulk API v.2.0 Query} which is available in API version 47.0 and later.
- * This API is subject to {@link https://sforce.co/4b6kn6z rate limits}.
- * @public
- * @example
- * <caption>The results will be available on `state.data`</caption>
- * bulkQuery(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);
- * @example
- * bulkQuery(
- *   (state) =>
- *     `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`,
- *   { pollTimeout: 10000, pollInterval: 6000 }
- * );
- * @function
- * @param {string} qs - A query string.
- * @param {object} options - Options passed to the bulk api.
- * @param {integer} [options.pollTimeout=90000] - Polling timeout in milliseconds.
- * @param {integer} [options.pollInterval=3000] - Polling interval in milliseconds.
- * @param {function} callback - A callback to execute once the record is retrieved
- * @returns {Operation}
- */
-export function bulkQuery(qs, options, callback) {
-  return async state => {
-    const { connection } = state;
-    const [resolvedQs, resolvedOptions] = newExpandReferences(
-      state,
-      qs,
-      options
-    );
-
-    if (parseFloat(connection.version) < 47.0)
-      throw new Error('bulkQuery requires API version 47.0 and later');
-
-    const { pollTimeout, pollInterval } = {
-      ...defaultOptions,
-      ...resolvedOptions,
-    };
-
-    console.log(`Executing query: ${resolvedQs}`);
-
-    const queryJob = await connection.request({
-      method: 'POST',
-      url: `/services/data/v${connection.version}/jobs/query`,
-      body: JSON.stringify({
-        operation: 'query',
-        query: resolvedQs,
-      }),
-      headers: {
-        'Content-Type': 'application/json',
-      },
-    });
-
-    const result = await pollJobResult(
-      connection,
-      queryJob,
-      pollInterval,
-      pollTimeout
-    );
-
-    const nextState = {
-      ...composeNextState(state, result),
-      result,
-    };
-    if (callback) return callback(nextState);
-
-    return nextState;
+    // Note: we no longer need `steps` anymore since `commonExecute`
+    // takes each operation as an argument.
+    return commonExecute(
+      loadAnyAscii,
+      util.createConnection,
+      ...flatten(operations),
+      util.removeConnection
+    )({ ...initialState, ...state });
   };
 }
 
@@ -493,6 +196,138 @@ export function bulk(sObject, operation, options, records) {
     });
   };
 }
+const defaultOptions = {
+  pollTimeout: 90000, // in ms
+  pollInterval: 3000, // in ms
+};
+/**
+ * Execute an SOQL Bulk Query.
+ * This function uses bulk query to efficiently query large data sets and reduce the number of API requests.
+ * `bulkQuery()` uses {@link https://sforce.co/4azgczz Bulk API v.2.0 Query} which is available in API version 47.0 and later.
+ * This API is subject to {@link https://sforce.co/4b6kn6z rate limits}.
+ * @public
+ * @example
+ * <caption>The results will be available on `state.data`</caption>
+ * bulkQuery(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);
+ * @example
+ * bulkQuery(
+ *   (state) =>
+ *     `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`,
+ *   { pollTimeout: 10000, pollInterval: 6000 }
+ * );
+ * @function
+ * @param {string} qs - A query string.
+ * @param {object} options - Options passed to the bulk api.
+ * @param {integer} [options.pollTimeout=90000] - Polling timeout in milliseconds.
+ * @param {integer} [options.pollInterval=3000] - Polling interval in milliseconds.
+ * @param {function} callback - A callback to execute once the record is retrieved
+ * @returns {Operation}
+ */
+export function bulkQuery(qs, options, callback) {
+  return async state => {
+    const { connection } = state;
+    const [resolvedQs, resolvedOptions] = newExpandReferences(
+      state,
+      qs,
+      options
+    );
+
+    if (parseFloat(connection.version) < 47.0)
+      throw new Error('bulkQuery requires API version 47.0 and later');
+
+    const { pollTimeout, pollInterval } = {
+      ...defaultOptions,
+      ...resolvedOptions,
+    };
+
+    console.log(`Executing query: ${resolvedQs}`);
+
+    const queryJob = await connection.request({
+      method: 'POST',
+      url: `/services/data/v${connection.version}/jobs/query`,
+      body: JSON.stringify({
+        operation: 'query',
+        query: resolvedQs,
+      }),
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+
+    const result = await util.pollJobResult(
+      connection,
+      queryJob,
+      pollInterval,
+      pollTimeout
+    );
+
+    const nextState = {
+      ...composeNextState(state, result),
+      result,
+    };
+    if (callback) return callback(nextState);
+
+    return nextState;
+  };
+}
+
+/**
+ * Create a new sObject record(s).
+ * @public
+ * @example <caption> Single record creation</caption>
+ * create("Account", { Name: "My Account #1" });
+ * @example <caption> Multiple records creation</caption>
+ * create("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
+ * @function
+ * @param {string} sObject - API name of the sObject.
+ * @param {object} attrs - Field attributes for the new record.
+ * @returns {Operation}
+ */
+export function create(sObject, attrs) {
+  return state => {
+    let { connection } = state;
+    const finalAttrs = expandReferences(attrs)(state);
+    console.info(`Creating ${sObject}`, finalAttrs);
+
+    return connection.create(sObject, finalAttrs).then(function (recordResult) {
+      console.log('Result : ' + JSON.stringify(recordResult));
+      return {
+        ...state,
+        references: [recordResult, ...state.references],
+      };
+    });
+  };
+}
+
+/**
+ * Prints an sObject metadata and pushes the result to state.references
+ * @public
+ * @example
+ * describe('obj_name')
+ * @function
+ * @param {string} sObject - API name of the sObject.
+ * @returns {Operation}
+ */
+export function describe(sObject) {
+  return state => {
+    const { connection } = state;
+
+    const objectName = expandReferences(sObject)(state);
+
+    return connection
+      .sobject(objectName)
+      .describe()
+      .then(result => {
+        console.log('Label : ' + result.label);
+        console.log('Num of Fields : ' + result.fields.length);
+
+        return {
+          ...state,
+          references: [result, ...state.references],
+        };
+      });
+  };
+}
 
 /**
  * Delete records of an object.
@@ -540,33 +375,28 @@ export function destroy(sObject, attrs, options) {
 }
 
 /**
- * Create a new sObject record(s).
+ * Prints the total number of all available sObjects and pushes the result to `state.references`.
  * @public
- * @example <caption> Single record creation</caption>
- * create("Account", { Name: "My Account #1" });
- * @example <caption> Multiple records creation</caption>
- * create("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
- * @function
- * @param {string} sObject - API name of the sObject.
- * @param {object} attrs - Field attributes for the new record.
+ * @example
+ * describeAll()
+ * @function
  * @returns {Operation}
  */
-export function create(sObject, attrs) {
+export function describeAll() {
   return state => {
-    let { connection } = state;
-    const finalAttrs = expandReferences(attrs)(state);
-    console.info(`Creating ${sObject}`, finalAttrs);
+    const { connection } = state;
+
+    return connection.describeGlobal().then(result => {
+      const { sobjects } = result;
+      console.log(`Retrieved ${sobjects.length} sObjects`);
 
-    return connection.create(sObject, finalAttrs).then(function (recordResult) {
-      console.log('Result : ' + JSON.stringify(recordResult));
       return {
         ...state,
-        references: [recordResult, ...state.references],
+        references: [sobjects, ...state.references],
       };
     });
   };
 }
-
 /**
  * Alias for "create(sObject, attrs)".
  * @public
@@ -583,49 +413,95 @@ export function insert(sObject, attrs) {
   return create(sObject, attrs);
 }
 
+export function post(url, data, options) {
+  return state;
+}
+
 /**
- * Create a new sObject if conditions are met.
+ * Execute an SOQL query.
+ * Note that in an event of a query error,
+ * error logs will be printed but the operation will not throw the error.
  *
- * **The `createIf()` function has been deprecated. Use `fnIf(condition,create())` instead.**
+ * The Salesforce query API is subject to rate limits, {@link https://sforce.co/3W9zyaQ See for more details}.
  * @public
  * @example
- * createIf(true, 'obj_name', {
- *   attr1: "foo",
- *   attr2: "bar"
- * })
+ * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);
+ * @example <caption>Query more records if next records are available</caption>
+ * query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true });
  * @function
- * @param {boolean} logical - a logical statement that will be evaluated.
- * @param {string} sObject - API name of the sObject.
- * @param {(object|object[])} attrs - Field attributes for the new object.
+ * @param {string} qs - A query string. Must be less than `4000` characters in WHERE clause
+ * @param {object} options - Options passed to the bulk api.
+ * @param {boolean} [options.autoFetch=false] - Fetch next records if available.
+ * @param {function} callback - A callback to execute once the record is retrieved
  * @returns {Operation}
  */
-export function createIf(logical, sObject, attrs) {
-  return state => {
-    const resolvedLogical = expandReferences(logical)(state);
+export function query(qs, options, callback = s => s) {
+  return async state => {
+    let done = false;
+    let qResult = null;
+    let result = [];
 
-    console.warn(
-      `The 'createIf()' function has been deprecated. Use 'fnIf(condition,create())' instead.`
+    const { connection } = state;
+    const [resolvedQs, resolvedOptions] = newExpandReferences(
+      state,
+      qs,
+      options
     );
+    const { autoFetch } = { ...{ autoFetch: false }, ...resolvedOptions };
+
+    console.log(`Executing query: ${resolvedQs}`);
+    try {
+      qResult = await connection.query(resolvedQs);
+    } catch (err) {
+      const { message, errorCode } = err;
+      console.log(`Error ${errorCode}: ${message}`);
+      throw err;
+    }
+
+    if (qResult.totalSize > 0) {
+      console.log('Total records', qResult.totalSize);
+
+      while (!done) {
+        result.push(qResult);
+
+        if (qResult.done) {
+          done = true;
+        } else if (autoFetch) {
+          console.log(
+            'Fetched records so far',
+            result.map(ref => ref.records).flat().length
+          );
+          console.log('Fetching next records...');
+          try {
+            qResult = await connection.request({ url: qResult.nextRecordsUrl });
+          } catch (err) {
+            const { message, errorCode } = err;
+            console.log(`Error ${errorCode}: ${message}`);
+            throw err;
+          }
+        } else {
+          done = true;
+        }
+      }
 
-    if (resolvedLogical) {
-      const { connection } = state;
-      const finalAttrs = expandReferences(attrs)(state);
-      console.info(`Creating ${sObject}`, finalAttrs);
-      return connection
-        .create(sObject, finalAttrs)
-        .then(function (recordResult) {
-          console.log('Result : ' + JSON.stringify(recordResult));
-          return {
-            ...state,
-            references: [recordResult, ...state.references],
-          };
-        });
+      console.log(
+        'Done ✔ retrieved records',
+        result.map(ref => ref.records).flat().length
+      );
     } else {
-      console.info(`Not creating ${sObject} because logical is false.`);
-      return {
-        ...state,
-      };
+      result.push(qResult);
+      console.log('No records found.');
     }
+
+    console.log(
+      'Results retrieved and pushed to position [0] of the references array.'
+    );
+
+    const nextState = {
+      ...state,
+      references: [result, ...state.references],
+    };
+    return callback(nextState);
   };
 }
 
@@ -672,59 +548,6 @@ export function upsert(sObject, externalId, attrs) {
   };
 }
 
-/**
- * Conditionally create a new sObject record, or updates it if it already exists
- *
- * **The `upsertIf()` function has been deprecated. Use `fnIf(condition,upsert())` instead.**
- * @public
- * @example
- * upsertIf(true, 'obj_name', 'ext_id', {
- *   attr1: "foo",
- *   attr2: "bar"
- * })
- * @function
- * @param {boolean} logical - a logical statement that will be evaluated.
- * @param {string} sObject - API name of the sObject.
- * @param {string} externalId - ID.
- * @param {(object|object[])} attrs - Field attributes for the new object.
- * @returns {Operation}
- */
-export function upsertIf(logical, sObject, externalId, attrs) {
-  return state => {
-    const resolvedLogical = expandReferences(logical)(state);
-
-    console.warn(
-      `The 'upsertIf()' function has been deprecated. Use 'fnIf(condition,upsert())' instead.`
-    );
-
-    if (resolvedLogical) {
-      const { connection } = state;
-      const finalAttrs = expandReferences(attrs)(state);
-      console.info(
-        `Upserting ${sObject} with externalId`,
-        externalId,
-        ':',
-        finalAttrs
-      );
-
-      return connection
-        .upsert(sObject, finalAttrs, externalId)
-        .then(function (recordResult) {
-          console.log('Result : ' + JSON.stringify(recordResult));
-          return {
-            ...state,
-            references: [recordResult, ...state.references],
-          };
-        });
-    } else {
-      console.info(`Not upserting ${sObject} because logical is false.`);
-      return {
-        ...state,
-      };
-    }
-  };
-}
-
 /**
  * Update an sObject record or records.
  * @public
@@ -759,144 +582,6 @@ export function update(sObject, attrs) {
   };
 }
 
-/**
- * Get a reference ID by an index.
- * @public
- * @example
- * reference(0)
- * @function
- * @param {number} position - Position for references array.
- * @returns {State}
- */
-export function reference(position) {
-  return state => state.references[position].id;
-}
-
-function getConnection(state, options) {
-  const { apiVersion } = state.configuration;
-
-  const apiVersionRegex = /^\d{2}\.\d$/;
-
-  if (apiVersion && apiVersionRegex.test(apiVersion)) {
-    options.version = apiVersion;
-  } else {
-    options.version = '47.0';
-  }
-  console.log('Using Salesforce API version:', options.version);
-
-  return new jsforce.Connection(options);
-}
-
-async function createBasicAuthConnection(state) {
-  const { loginUrl, username, password, securityToken } = state.configuration;
-
-  const connection = getConnection(state, { loginUrl });
-
-  await connection
-    .login(username, securityToken ? password + securityToken : password)
-    .catch(e => {
-      console.error(`Failed to connect to salesforce as ${username}`);
-      throw e;
-    });
-
-  console.info(`Connected to salesforce as ${username}.`);
-
-  return {
-    ...state,
-    connection,
-  };
-}
-
-function createAccessTokenConnection(state) {
-  const { instance_url, access_token } = state.configuration;
-
-  const connection = getConnection(state, {
-    instanceUrl: instance_url,
-    accessToken: access_token,
-  });
-
-  console.log(`Connected with ${connection._sessionType} session type`);
-
-  return {
-    ...state,
-    connection,
-  };
-}
-
-/**
- * Creates a connection to Salesforce using Basic Auth or OAuth.
- * @function createConnection
- * @private
- * @param {State} state - Runtime state.
- * @returns {State}
- */
-function createConnection(state) {
-  if (state.connection) {
-    return state;
-  }
-
-  const { access_token } = state.configuration;
-
-  return access_token
-    ? createAccessTokenConnection(state)
-    : createBasicAuthConnection(state);
-}
-
-/**
- * Executes an operation.
- * @function
- * @param {Operation} operations - Operations
- * @returns {State}
- */
-export function execute(...operations) {
-  const initialState = {
-    logger: {
-      info: console.info.bind(console),
-      debug: console.log.bind(console),
-    },
-    references: [],
-    data: null,
-    configuration: {},
-  };
-
-  return state => {
-    // Note: we no longer need `steps` anymore since `commonExecute`
-    // takes each operation as an argument.
-    return commonExecute(
-      loadAnyAscii,
-      createConnection,
-      ...flatten(operations),
-      cleanupState
-    )({ ...initialState, ...state });
-  };
-}
-/**
- * Removes unserializable keys from the state.
- * @example
- * cleanupState(state)
- * @function
- * @param {State} state
- * @returns {State}
- */
-function cleanupState(state) {
-  delete state.connection;
-  return state;
-}
-
-/**
- * Flattens an array of operations.
- * @example
- * steps(
- *   createIf(params),
- *   update(params)
- * )
- * @function
- * @returns {array}
- */
-export function steps(...operations) {
-  return flatten(operations);
-}
-
 /**
  * Transliterates unicode characters to their best ASCII representation
  * @public
@@ -957,15 +642,68 @@ export function request(path, options, callback = s => s) {
     return callback(nextState);
   };
 }
-// Note that we expose the entire axios package to the user here.
-import axios from 'axios';
 
-export { axios };
+/**
+ * Retrieves a Salesforce sObject(s).
+ * @public
+ * @example
+ * retrieve('ContentVersion', '0684K0000020Au7QAE/VersionData');
+ * @function
+ * @param {string} sObject - The sObject to retrieve
+ * @param {string} id - The id of the record
+ * @param {function} callback - A callback to execute once the record is retrieved
+ * @returns {Operation}
+ */
+export function retrieve(sObject, id, callback) {
+  return state => {
+    const { connection } = state;
+
+    const finalId = expandReferences(id)(state);
+
+    return connection
+      .sobject(sObject)
+      .retrieve(finalId)
+      .then(result => {
+        return {
+          ...state,
+          references: [result, ...state.references],
+        };
+      })
+      .then(state => {
+        if (callback) {
+          return callback(state);
+        }
+        return state;
+      });
+  };
+}
+
+/**
+ * Adds a lookup relation or 'dome insert' to a record.
+ * @public
+ * @example
+ * Data Sourced Value:
+ *  relationship("relationship_name__r", "externalID on related object", dataSource("path"))
+ * Fixed Value:
+ *  relationship("relationship_name__r", "externalID on related object", "hello world")
+ * @function
+ * @param {string} relationshipName - `__r` relationship field on the record.
+ * @param {string} externalId - Salesforce ExternalID field.
+ * @param {string} dataSource - resolvable source.
+ * @returns {object}
+ */
+export function relationship(relationshipName, externalId, dataSource) {
+  return field(relationshipName, state => {
+    if (typeof dataSource == 'function') {
+      return { [externalId]: dataSource(state) };
+    }
+    return { [externalId]: dataSource };
+  });
+}
 
 export {
   alterState,
   arrayToString,
-  beta,
   chunk,
   combine,
   dataPath,
diff --git a/packages/salesforce/test/Adaptor.test.js b/packages/salesforce/test/Adaptor.test.js
index 46120a1cf..9816ae07a 100644
--- a/packages/salesforce/test/Adaptor.test.js
+++ b/packages/salesforce/test/Adaptor.test.js
@@ -1,26 +1,10 @@
 import chai from 'chai';
 import sinon from 'sinon';
-import {
-  reference,
-  create,
-  createIf,
-  upsert,
-  upsertIf,
-  toUTF8,
-  execute,
-} from '../src/Adaptor';
+import { create, upsert, toUTF8, execute } from '../src/Adaptor';
 
 const { expect } = chai;
 
 describe('Adaptor', () => {
-  describe('reference', () => {
-    it('returns the Id of a previous operation', () => {
-      let state = { references: [{ id: '12345' }] };
-      let Id = reference(0)(state);
-      expect(Id).to.eql('12345');
-    });
-  });
-
   describe('create', () => {
     it('makes a new sObject', done => {
       const fakeConnection = {
@@ -49,59 +33,6 @@ describe('Adaptor', () => {
     });
   });
 
-  describe('createIf', () => {
-    it("doesn't create a new sObject if a logical is false", done => {
-      const fakeConnection = {
-        create: function () {
-          return Promise.resolve({ Id: 10 });
-        },
-      };
-      let state = { connection: fakeConnection, references: [] };
-
-      let logical = 1 + 1 == 3;
-
-      let sObject = 'myObject';
-      let fields = { field: 'value' };
-
-      let spy = sinon.spy(fakeConnection, 'create');
-
-      createIf(logical, sObject, fields)(state);
-
-      expect(spy.called).to.eql(false);
-      expect(state).to.eql({ connection: fakeConnection, references: [] });
-      done();
-    });
-
-    it('makes a new sObject if a logical is true', done => {
-      const fakeConnection = {
-        create: function () {
-          return Promise.resolve({ Id: 10 });
-        },
-      };
-      let state = { connection: fakeConnection, references: [] };
-
-      let logical = 1 + 1 == 2;
-
-      let sObject = 'myObject';
-      let fields = { field: 'value' };
-
-      let spy = sinon.spy(fakeConnection, 'create');
-
-      createIf(
-        logical,
-        sObject,
-        fields
-      )(state)
-        .then(state => {
-          expect(spy.args[0]).to.eql([sObject, fields]);
-          expect(spy.called).to.eql(true);
-          expect(state.references[0]).to.eql({ Id: 10 });
-        })
-        .then(done)
-        .catch(done);
-    });
-  });
-
   describe('upsert', () => {
     it('is expected to call `upsert` on the connection', done => {
       const connection = {
@@ -132,39 +63,6 @@ describe('Adaptor', () => {
     });
   });
 
-  describe('upsertIf', () => {
-    it('upserts if a logical is true', done => {
-      const fakeConnection = {
-        upsert: function () {
-          return Promise.resolve({ Id: 10 });
-        },
-      };
-      let state = { connection: fakeConnection, references: [] };
-
-      let logical = 1 + 1 == 2;
-
-      let sObject = 'myObject';
-      let externalId = 'MyExternalId';
-      let fields = { field: 'value' };
-
-      let spy = sinon.spy(fakeConnection, 'upsert');
-
-      upsertIf(
-        logical,
-        sObject,
-        externalId,
-        fields
-      )(state)
-        .then(state => {
-          expect(spy.args[0]).to.eql([sObject, fields, externalId]);
-          expect(spy.called).to.eql(true);
-          expect(state.references[0]).to.eql({ Id: 10 });
-        })
-        .then(done)
-        .catch(done);
-    });
-  });
-
   describe('toUTF8', () => {
     it('Transliterate unicode to ASCII representation', async () => {
       const state = {

From fe8b1e00ef5bed309cbdd9fa25e990fa7bd52960 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Wed, 17 Jul 2024 15:50:51 +0300
Subject: [PATCH 04/22] wip prepareNextState

---
 packages/salesforce/ast.json       | 606 +++++++++++------------------
 packages/salesforce/src/Adaptor.js | 111 +++---
 packages/salesforce/src/Utils.js   |  12 +
 3 files changed, 284 insertions(+), 445 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index ed54ae5ed..a1ba836d6 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -1,73 +1,15 @@
 {
   "operations": [
     {
-      "name": "relationship",
+      "name": "bulk",
       "params": [
-        "relationshipName",
-        "externalId",
-        "dataSource"
+        "sObject",
+        "operation",
+        "records",
+        "options"
       ],
       "docs": {
-        "description": "Adds a lookup relation or 'dome insert' to a record.",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "Data Sourced Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", dataSource(\"path\"))\nFixed Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", \"hello world\")"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
-          {
-            "title": "param",
-            "description": "`__r` relationship field on the record.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "relationshipName"
-          },
-          {
-            "title": "param",
-            "description": "Salesforce ExternalID field.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "externalId"
-          },
-          {
-            "title": "param",
-            "description": "resolvable source.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "dataSource"
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "object"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
-    {
-      "name": "describeAll",
-      "params": [],
-      "docs": {
-        "description": "Prints the total number of all available sObjects and pushes the result to `state.references`.",
+        "description": "Create and execute a bulk job.",
         "tags": [
           {
             "title": "public",
@@ -76,41 +18,13 @@
           },
           {
             "title": "example",
-            "description": "describeAll()"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "Operation"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
-    {
-      "name": "describe",
-      "params": [
-        "sObject"
-      ],
-      "docs": {
-        "description": "Prints an sObject metadata and pushes the result to state.references",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
+            "description": "bulk(\n  \"Patient__c\",\n  \"insert\",\n  (state) => state.patients.map((x) => ({ Age__c: x.age, Name: x.name })),\n  { failOnError: true }\n);",
+            "caption": "Bulk insert"
           },
           {
             "title": "example",
-            "description": "describe('obj_name')"
+            "description": "bulk(\n  \"vera__Beneficiary__c\",\n  \"upsert\",\n  [\n    {\n      vera__Reporting_Period__c: 2023,\n      vera__Geographic_Area__c: \"Uganda\",\n      \"vera__Indicator__r.vera__ExtId__c\": 1001,\n      vera__Result_UID__c: \"1001_2023_Uganda\",\n    },\n  ],\n  { extIdField: \"vera__Result_UID__c\" }\n);",
+            "caption": "Bulk upsert"
           },
           {
             "title": "function",
@@ -126,149 +40,83 @@
             },
             "name": "sObject"
           },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "Operation"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
-    {
-      "name": "retrieve",
-      "params": [
-        "sObject",
-        "id",
-        "callback"
-      ],
-      "docs": {
-        "description": "Retrieves a Salesforce sObject(s).",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "retrieve('ContentVersion', '0684K0000020Au7QAE/VersionData');"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
           {
             "title": "param",
-            "description": "The sObject to retrieve",
+            "description": "The bulk operation to be performed.Eg \"insert\" | \"update\" | \"upsert\"",
             "type": {
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "operation"
           },
           {
             "title": "param",
-            "description": "The id of the record",
+            "description": "an array of records, or a function which returns an array.",
             "type": {
               "type": "NameExpression",
-              "name": "string"
+              "name": "array"
             },
-            "name": "id"
+            "name": "records"
           },
           {
             "title": "param",
-            "description": "A callback to execute once the record is retrieved",
+            "description": "Options passed to the bulk api.",
             "type": {
               "type": "NameExpression",
-              "name": "function"
+              "name": "object"
             },
-            "name": "callback"
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "Operation"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
-    {
-      "name": "query",
-      "params": [
-        "qs",
-        "options",
-        "callback"
-      ],
-      "docs": {
-        "description": "Execute an SOQL query.\nNote that in an event of a query error,\nerror logs will be printed but the operation will not throw the error.\n\nThe Salesforce query API is subject to rate limits, {@link https://sforce.co/3W9zyaQ See for more details}.",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);"
-          },
-          {
-            "title": "example",
-            "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true });",
-            "caption": "Query more records if next records are available"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
+            "name": "options"
           },
           {
             "title": "param",
-            "description": "A query string. Must be less than `4000` characters in WHERE clause",
+            "description": "External id field.",
             "type": {
-              "type": "NameExpression",
-              "name": "string"
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "string"
+              }
             },
-            "name": "qs"
+            "name": "options.extIdField"
           },
           {
             "title": "param",
-            "description": "Options passed to the bulk api.",
+            "description": "Fail the operation on error.",
             "type": {
-              "type": "NameExpression",
-              "name": "object"
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "boolean"
+              }
             },
-            "name": "options"
+            "name": "options.failOnError",
+            "default": "false"
           },
           {
             "title": "param",
-            "description": "Fetch next records if available.",
+            "description": "Polling interval in milliseconds.",
             "type": {
               "type": "OptionalType",
               "expression": {
                 "type": "NameExpression",
-                "name": "boolean"
+                "name": "integer"
               }
             },
-            "name": "options.autoFetch",
-            "default": "false"
+            "name": "options.pollInterval",
+            "default": "6000"
           },
           {
             "title": "param",
-            "description": "A callback to execute once the record is retrieved",
+            "description": "Polling timeout in milliseconds.",
             "type": {
-              "type": "NameExpression",
-              "name": "function"
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "integer"
+              }
             },
-            "name": "callback"
+            "name": "options.pollTimeout",
+            "default": "240000"
           },
           {
             "title": "returns",
@@ -286,8 +134,7 @@
       "name": "bulkQuery",
       "params": [
         "qs",
-        "options",
-        "callback"
+        "options"
       ],
       "docs": {
         "description": "Execute an SOQL Bulk Query.\nThis function uses bulk query to efficiently query large data sets and reduce the number of API requests.\n`bulkQuery()` uses {@link https://sforce.co/4azgczz Bulk API v.2.0 Query} which is available in API version 47.0 and later.\nThis API is subject to {@link https://sforce.co/4b6kn6z rate limits}.",
@@ -355,15 +202,6 @@
             "name": "options.pollInterval",
             "default": "3000"
           },
-          {
-            "title": "param",
-            "description": "A callback to execute once the record is retrieved",
-            "type": {
-              "type": "NameExpression",
-              "name": "function"
-            },
-            "name": "callback"
-          },
           {
             "title": "returns",
             "description": null,
@@ -377,15 +215,13 @@
       "valid": false
     },
     {
-      "name": "bulk",
+      "name": "create",
       "params": [
         "sObject",
-        "operation",
-        "options",
         "records"
       ],
       "docs": {
-        "description": "Create and execute a bulk job.",
+        "description": "Create a new sObject record(s).",
         "tags": [
           {
             "title": "public",
@@ -394,13 +230,13 @@
           },
           {
             "title": "example",
-            "description": "bulk(\n  \"Patient__c\",\n  \"insert\",\n  { failOnError: true },\n  (state) => state.someArray.map((x) => ({ Age__c: x.age, Name: x.name }))\n);",
-            "caption": "Bulk insert"
+            "description": "create(\"Account\", { Name: \"My Account #1\" });",
+            "caption": "Single record creation"
           },
           {
             "title": "example",
-            "description": "bulk(\n  \"vera__Beneficiary__c\",\n  \"upsert\",\n  { extIdField: \"vera__Result_UID__c\" },\n  [\n    {\n      vera__Reporting_Period__c: 2023,\n      vera__Geographic_Area__c: \"Uganda\",\n      \"vera__Indicator__r.vera__ExtId__c\": 1001,\n      vera__Result_UID__c: \"1001_2023_Uganda\",\n    },\n  ]\n);",
-            "caption": "Bulk upsert"
+            "description": "create(\"Account\",[{ Name: \"My Account #1\" }, { Name: \"My Account #2\" }]);",
+            "caption": "Multiple records creation"
           },
           {
             "title": "function",
@@ -418,81 +254,55 @@
           },
           {
             "title": "param",
-            "description": "The bulk operation to be performed.Eg \"insert\" | \"update\" | \"upsert\"",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "operation"
-          },
-          {
-            "title": "param",
-            "description": "Options passed to the bulk api.",
+            "description": "Field attributes for the new record.",
             "type": {
               "type": "NameExpression",
               "name": "object"
             },
-            "name": "options"
+            "name": "records"
           },
           {
-            "title": "param",
-            "description": "Polling timeout in milliseconds.",
+            "title": "returns",
+            "description": null,
             "type": {
-              "type": "OptionalType",
-              "expression": {
-                "type": "NameExpression",
-                "name": "integer"
-              }
-            },
-            "name": "options.pollTimeout",
-            "default": "240000"
-          },
+              "type": "NameExpression",
+              "name": "Operation"
+            }
+          }
+        ]
+      },
+      "valid": true
+    },
+    {
+      "name": "describe",
+      "params": [
+        "sObject"
+      ],
+      "docs": {
+        "description": "Prints an sObject metadata and pushes the result to state.references",
+        "tags": [
           {
-            "title": "param",
-            "description": "Polling interval in milliseconds.",
-            "type": {
-              "type": "OptionalType",
-              "expression": {
-                "type": "NameExpression",
-                "name": "integer"
-              }
-            },
-            "name": "options.pollInterval",
-            "default": "6000"
+            "title": "public",
+            "description": null,
+            "type": null
           },
-          {
-            "title": "param",
-            "description": "External id field.",
-            "type": {
-              "type": "OptionalType",
-              "expression": {
-                "type": "NameExpression",
-                "name": "string"
-              }
-            },
-            "name": "options.extIdField"
+          {
+            "title": "example",
+            "description": "describe('obj_name')"
           },
           {
-            "title": "param",
-            "description": "Fail the operation on error.",
-            "type": {
-              "type": "OptionalType",
-              "expression": {
-                "type": "NameExpression",
-                "name": "boolean"
-              }
-            },
-            "name": "options.failOnError",
-            "default": "false"
+            "title": "function",
+            "description": null,
+            "name": null
           },
           {
             "title": "param",
-            "description": "an array of records, or a function which returns an array.",
+            "description": "API name of the sObject.",
             "type": {
               "type": "NameExpression",
-              "name": "array"
+              "name": "string"
             },
-            "name": "records"
+            "name": "sObject"
           },
           {
             "title": "returns",
@@ -504,7 +314,7 @@
           }
         ]
       },
-      "valid": false
+      "valid": true
     },
     {
       "name": "destroy",
@@ -570,13 +380,45 @@
       "valid": true
     },
     {
-      "name": "create",
+      "name": "describeAll",
+      "params": [],
+      "docs": {
+        "description": "Prints the total number of all available sObjects and pushes the result to `state.references`.",
+        "tags": [
+          {
+            "title": "public",
+            "description": null,
+            "type": null
+          },
+          {
+            "title": "example",
+            "description": "describeAll()"
+          },
+          {
+            "title": "function",
+            "description": null,
+            "name": null
+          },
+          {
+            "title": "returns",
+            "description": null,
+            "type": {
+              "type": "NameExpression",
+              "name": "Operation"
+            }
+          }
+        ]
+      },
+      "valid": true
+    },
+    {
+      "name": "insert",
       "params": [
         "sObject",
-        "attrs"
+        "records"
       ],
       "docs": {
-        "description": "Create a new sObject record(s).",
+        "description": "Alias for \"create(sObject, attrs)\".",
         "tags": [
           {
             "title": "public",
@@ -585,12 +427,12 @@
           },
           {
             "title": "example",
-            "description": "create(\"Account\", { Name: \"My Account #1\" });",
+            "description": "insert(\"Account\", { Name: \"My Account #1\" });",
             "caption": "Single record creation"
           },
           {
             "title": "example",
-            "description": "create(\"Account\",[{ Name: \"My Account #1\" }, { Name: \"My Account #2\" }]);",
+            "description": "insert(\"Account\",[{ Name: \"My Account #1\" }, { Name: \"My Account #2\" }]);",
             "caption": "Multiple records creation"
           },
           {
@@ -614,7 +456,7 @@
               "type": "NameExpression",
               "name": "object"
             },
-            "name": "attrs"
+            "name": "records"
           },
           {
             "title": "returns",
@@ -629,13 +471,13 @@
       "valid": true
     },
     {
-      "name": "insert",
+      "name": "query",
       "params": [
-        "sObject",
-        "attrs"
+        "qs",
+        "options"
       ],
       "docs": {
-        "description": "Alias for \"create(sObject, attrs)\".",
+        "description": "Execute an SOQL query.\nNote that in an event of a query error,\nerror logs will be printed but the operation will not throw the error.\n\nThe Salesforce query API is subject to rate limits, {@link https://sforce.co/3W9zyaQ See for more details}.",
         "tags": [
           {
             "title": "public",
@@ -644,13 +486,12 @@
           },
           {
             "title": "example",
-            "description": "insert(\"Account\", { Name: \"My Account #1\" });",
-            "caption": "Single record creation"
+            "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`);"
           },
           {
             "title": "example",
-            "description": "insert(\"Account\",[{ Name: \"My Account #1\" }, { Name: \"My Account #2\" }]);",
-            "caption": "Multiple records creation"
+            "description": "query(state=> `SELECT Id FROM Patient__c WHERE Health_ID__c = '${state.data.field1}'`, { autoFetch: true });",
+            "caption": "Query more records if next records are available"
           },
           {
             "title": "function",
@@ -659,21 +500,34 @@
           },
           {
             "title": "param",
-            "description": "API name of the sObject.",
+            "description": "A query string. Must be less than `4000` characters in WHERE clause",
             "type": {
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "qs"
           },
           {
             "title": "param",
-            "description": "Field attributes for the new record.",
+            "description": "Options passed to the bulk api.",
             "type": {
               "type": "NameExpression",
               "name": "object"
             },
-            "name": "attrs"
+            "name": "options"
+          },
+          {
+            "title": "param",
+            "description": "Fetch next records if available.",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "boolean"
+              }
+            },
+            "name": "options.autoFetch",
+            "default": "false"
           },
           {
             "title": "returns",
@@ -685,7 +539,7 @@
           }
         ]
       },
-      "valid": true
+      "valid": false
     },
     {
       "name": "upsert",
@@ -787,15 +641,13 @@
       "valid": true
     },
     {
-      "name": "upsertIf",
+      "name": "update",
       "params": [
-        "logical",
         "sObject",
-        "externalId",
         "attrs"
       ],
       "docs": {
-        "description": "Conditionally create a new sObject record, or updates it if it already exists\n\n**The `upsertIf()` function has been deprecated. Use `fnIf(condition,upsert())` instead.**",
+        "description": "Update an sObject record or records.",
         "tags": [
           {
             "title": "public",
@@ -804,22 +656,19 @@
           },
           {
             "title": "example",
-            "description": "upsertIf(true, 'obj_name', 'ext_id', {\n  attr1: \"foo\",\n  attr2: \"bar\"\n})"
+            "description": "update(\"Account\", {\n  Id: \"0010500000fxbcuAAA\",\n  Name: \"Updated Account #1\",\n});",
+            "caption": "Single record update"
+          },
+          {
+            "title": "example",
+            "description": "update(\"Account\", [\n  { Id: \"0010500000fxbcuAAA\", Name: \"Updated Account #1\" },\n  { Id: \"0010500000fxbcvAAA\", Name: \"Updated Account #2\" },\n]);",
+            "caption": "Multiple records update"
           },
           {
             "title": "function",
             "description": null,
             "name": null
           },
-          {
-            "title": "param",
-            "description": "a logical statement that will be evaluated.",
-            "type": {
-              "type": "NameExpression",
-              "name": "boolean"
-            },
-            "name": "logical"
-          },
           {
             "title": "param",
             "description": "API name of the sObject.",
@@ -829,15 +678,6 @@
             },
             "name": "sObject"
           },
-          {
-            "title": "param",
-            "description": "ID.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "externalId"
-          },
           {
             "title": "param",
             "description": "Field attributes for the new object.",
@@ -878,13 +718,12 @@
       "valid": true
     },
     {
-      "name": "update",
+      "name": "toUTF8",
       "params": [
-        "sObject",
-        "attrs"
+        "input"
       ],
       "docs": {
-        "description": "Update an sObject record or records.",
+        "description": "Transliterates unicode characters to their best ASCII representation",
         "tags": [
           {
             "title": "public",
@@ -893,61 +732,23 @@
           },
           {
             "title": "example",
-            "description": "update(\"Account\", {\n  Id: \"0010500000fxbcuAAA\",\n  Name: \"Updated Account #1\",\n});",
-            "caption": "Single record update"
-          },
-          {
-            "title": "example",
-            "description": "update(\"Account\", [\n  { Id: \"0010500000fxbcuAAA\", Name: \"Updated Account #1\" },\n  { Id: \"0010500000fxbcvAAA\", Name: \"Updated Account #2\" },\n]);",
-            "caption": "Multiple records update"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
+            "description": "fn((state) => {\n  const s = toUTF8(\"άνθρωποι\");\n  console.log(s); // anthropoi\n  return state;\n});"
           },
           {
             "title": "param",
-            "description": "API name of the sObject.",
+            "description": "A string with unicode characters",
             "type": {
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
-          },
-          {
-            "title": "param",
-            "description": "Field attributes for the new object.",
-            "type": {
-              "type": "UnionType",
-              "elements": [
-                {
-                  "type": "NameExpression",
-                  "name": "object"
-                },
-                {
-                  "type": "TypeApplication",
-                  "expression": {
-                    "type": "NameExpression",
-                    "name": "Array"
-                  },
-                  "applications": [
-                    {
-                      "type": "NameExpression",
-                      "name": "object"
-                    }
-                  ]
-                }
-              ]
-            },
-            "name": "attrs"
+            "name": "input"
           },
           {
             "title": "returns",
-            "description": null,
+            "description": "ASCII representation of input string",
             "type": {
               "type": "NameExpression",
-              "name": "Operation"
+              "name": "string"
             }
           }
         ]
@@ -955,12 +756,14 @@
       "valid": true
     },
     {
-      "name": "reference",
+      "name": "retrieve",
       "params": [
-        "position"
+        "sObject",
+        "id",
+        "callback"
       ],
       "docs": {
-        "description": "Get a reference ID by an index.",
+        "description": "Retrieves a Salesforce sObject(s).",
         "tags": [
           {
             "title": "public",
@@ -969,7 +772,7 @@
           },
           {
             "title": "example",
-            "description": "reference(0)"
+            "description": "retrieve('ContentVersion', '0684K0000020Au7QAE/VersionData');"
           },
           {
             "title": "function",
@@ -978,19 +781,37 @@
           },
           {
             "title": "param",
-            "description": "Position for references array.",
+            "description": "The sObject to retrieve",
+            "type": {
+              "type": "NameExpression",
+              "name": "string"
+            },
+            "name": "sObject"
+          },
+          {
+            "title": "param",
+            "description": "The id of the record",
+            "type": {
+              "type": "NameExpression",
+              "name": "string"
+            },
+            "name": "id"
+          },
+          {
+            "title": "param",
+            "description": "A callback to execute once the record is retrieved",
             "type": {
               "type": "NameExpression",
-              "name": "number"
+              "name": "function"
             },
-            "name": "position"
+            "name": "callback"
           },
           {
             "title": "returns",
             "description": null,
             "type": {
               "type": "NameExpression",
-              "name": "State"
+              "name": "Operation"
             }
           }
         ]
@@ -998,12 +819,14 @@
       "valid": true
     },
     {
-      "name": "toUTF8",
+      "name": "relationship",
       "params": [
-        "input"
+        "relationshipName",
+        "externalId",
+        "dataSource"
       ],
       "docs": {
-        "description": "Transliterates unicode characters to their best ASCII representation",
+        "description": "Adds a lookup relation or 'dome insert' to a record.",
         "tags": [
           {
             "title": "public",
@@ -1012,23 +835,46 @@
           },
           {
             "title": "example",
-            "description": "fn((state) => {\n  const s = toUTF8(\"άνθρωποι\");\n  console.log(s); // anthropoi\n  return state;\n});"
+            "description": "Data Sourced Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", dataSource(\"path\"))\nFixed Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", \"hello world\")"
+          },
+          {
+            "title": "function",
+            "description": null,
+            "name": null
           },
           {
             "title": "param",
-            "description": "A string with unicode characters",
+            "description": "`__r` relationship field on the record.",
             "type": {
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "input"
+            "name": "relationshipName"
           },
           {
-            "title": "returns",
-            "description": "ASCII representation of input string",
+            "title": "param",
+            "description": "Salesforce ExternalID field.",
+            "type": {
+              "type": "NameExpression",
+              "name": "string"
+            },
+            "name": "externalId"
+          },
+          {
+            "title": "param",
+            "description": "resolvable source.",
             "type": {
               "type": "NameExpression",
               "name": "string"
+            },
+            "name": "dataSource"
+          },
+          {
+            "title": "returns",
+            "description": null,
+            "type": {
+              "type": "NameExpression",
+              "name": "object"
             }
           }
         ]
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 5457289fb..e835875e7 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -70,14 +70,13 @@ export function execute(...operations) {
  * bulk(
  *   "Patient__c",
  *   "insert",
- *   { failOnError: true },
- *   (state) => state.someArray.map((x) => ({ Age__c: x.age, Name: x.name }))
+ *   (state) => state.patients.map((x) => ({ Age__c: x.age, Name: x.name })),
+ *   { failOnError: true }
  * );
  * @example <caption>Bulk upsert</caption>
  * bulk(
  *   "vera__Beneficiary__c",
  *   "upsert",
- *   { extIdField: "vera__Result_UID__c" },
  *   [
  *     {
  *       vera__Reporting_Period__c: 2023,
@@ -85,35 +84,36 @@ export function execute(...operations) {
  *       "vera__Indicator__r.vera__ExtId__c": 1001,
  *       vera__Result_UID__c: "1001_2023_Uganda",
  *     },
- *   ]
+ *   ],
+ *   { extIdField: "vera__Result_UID__c" }
  * );
  * @function
  * @param {string} sObject - API name of the sObject.
  * @param {string} operation - The bulk operation to be performed.Eg "insert" | "update" | "upsert"
+ * @param {array} records - an array of records, or a function which returns an array.
  * @param {object} options - Options passed to the bulk api.
- * @param {integer} [options.pollTimeout=240000] - Polling timeout in milliseconds.
- * @param {integer} [options.pollInterval=6000] - Polling interval in milliseconds.
  * @param {string} [options.extIdField] - External id field.
  * @param {boolean} [options.failOnError=false] - Fail the operation on error.
- * @param {array} records - an array of records, or a function which returns an array.
+ * @param {integer} [options.pollInterval=6000] - Polling interval in milliseconds.
+ * @param {integer} [options.pollTimeout=240000] - Polling timeout in milliseconds.
  * @returns {Operation}
  */
-export function bulk(sObject, operation, options, records) {
+export function bulk(sObject, operation, records, options) {
   return state => {
     const { connection } = state;
 
     const [
       resolvedSObject,
       resolvedOperation,
-      resolvedOptions,
       resolvedRecords,
+      resolvedOptions,
     ] = newExpandReferences(state, sObject, operation, options, records);
 
     const {
       failOnError = false,
       allowNoOp = false,
-      pollTimeout,
-      pollInterval,
+      pollTimeout = 240000,
+      pollInterval = 6000,
     } = resolvedOptions;
 
     if (allowNoOp && resolvedRecords.length === 0) {
@@ -192,14 +192,15 @@ export function bulk(sObject, operation, options, records) {
     ).then(arrayOfResults => {
       console.log('Merging results arrays.');
       const merged = [].concat.apply([], arrayOfResults);
+      return util.prepareNextState(state, merged);
       return { ...state, references: [merged, ...state.references] };
     });
   };
 }
-const defaultOptions = {
-  pollTimeout: 90000, // in ms
-  pollInterval: 3000, // in ms
-};
+// const defaultOptions = {
+//   pollTimeout: 90000, // in ms
+//   pollInterval: 3000, // in ms
+// };
 /**
  * Execute an SOQL Bulk Query.
  * This function uses bulk query to efficiently query large data sets and reduce the number of API requests.
@@ -220,25 +221,20 @@ const defaultOptions = {
  * @param {object} options - Options passed to the bulk api.
  * @param {integer} [options.pollTimeout=90000] - Polling timeout in milliseconds.
  * @param {integer} [options.pollInterval=3000] - Polling interval in milliseconds.
- * @param {function} callback - A callback to execute once the record is retrieved
  * @returns {Operation}
  */
-export function bulkQuery(qs, options, callback) {
+export function bulkQuery(qs, options) {
   return async state => {
     const { connection } = state;
-    const [resolvedQs, resolvedOptions] = newExpandReferences(
-      state,
-      qs,
-      options
-    );
+    const [
+      resolvedQs,
+      resolvedOptions = { pollTimeout: 90000, pollInterval: 3000 },
+    ] = newExpandReferences(state, qs, options);
 
     if (parseFloat(connection.version) < 47.0)
       throw new Error('bulkQuery requires API version 47.0 and later');
 
-    const { pollTimeout, pollInterval } = {
-      ...defaultOptions,
-      ...resolvedOptions,
-    };
+    const { pollTimeout, pollInterval } = resolvedOptions;
 
     console.log(`Executing query: ${resolvedQs}`);
 
@@ -261,13 +257,7 @@ export function bulkQuery(qs, options, callback) {
       pollTimeout
     );
 
-    const nextState = {
-      ...composeNextState(state, result),
-      result,
-    };
-    if (callback) return callback(nextState);
-
-    return nextState;
+    return util.prepareNextState(state, result);
   };
 }
 
@@ -280,22 +270,25 @@ export function bulkQuery(qs, options, callback) {
  * create("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
  * @function
  * @param {string} sObject - API name of the sObject.
- * @param {object} attrs - Field attributes for the new record.
+ * @param {object} records - Field attributes for the new record.
  * @returns {Operation}
  */
-export function create(sObject, attrs) {
+export function create(sObject, records) {
   return state => {
     let { connection } = state;
-    const finalAttrs = expandReferences(attrs)(state);
-    console.info(`Creating ${sObject}`, finalAttrs);
+    const [resolvedSObject, resolvedRecords] = newExpandReferences(
+      state,
+      sObject,
+      records
+    );
+    console.info(`Creating ${resolvedSObject}`, resolvedRecords);
 
-    return connection.create(sObject, finalAttrs).then(function (recordResult) {
-      console.log('Result : ' + JSON.stringify(recordResult));
-      return {
-        ...state,
-        references: [recordResult, ...state.references],
-      };
-    });
+    return connection
+      .create(resolvedSObject, resolvedRecords)
+      .then(recordResult => {
+        console.log('Result : ' + JSON.stringify(recordResult));
+        return util.prepareNextState(state, recordResult);
+      });
   };
 }
 
@@ -312,19 +305,16 @@ export function describe(sObject) {
   return state => {
     const { connection } = state;
 
-    const objectName = expandReferences(sObject)(state);
+    const [resolvedSObject] = newExpandReferences(state, sObject);
 
     return connection
-      .sobject(objectName)
+      .sobject(resolvedSObject)
       .describe()
       .then(result => {
         console.log('Label : ' + result.label);
         console.log('Num of Fields : ' + result.fields.length);
 
-        return {
-          ...state,
-          references: [result, ...state.references],
-        };
+        return util.prepareNextState(state, result);
       });
   };
 }
@@ -389,11 +379,7 @@ export function describeAll() {
     return connection.describeGlobal().then(result => {
       const { sobjects } = result;
       console.log(`Retrieved ${sobjects.length} sObjects`);
-
-      return {
-        ...state,
-        references: [sobjects, ...state.references],
-      };
+      return util.prepareNextState(state, { records: sobjects, ...result });
     });
   };
 }
@@ -406,11 +392,11 @@ export function describeAll() {
  * insert("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
  * @function
  * @param {string} sObject - API name of the sObject.
- * @param {object} attrs - Field attributes for the new record.
+ * @param {object} records - Field attributes for the new record.
  * @returns {Operation}
  */
-export function insert(sObject, attrs) {
-  return create(sObject, attrs);
+export function insert(sObject, records) {
+  return create(sObject, records);
 }
 
 export function post(url, data, options) {
@@ -432,10 +418,9 @@ export function post(url, data, options) {
  * @param {string} qs - A query string. Must be less than `4000` characters in WHERE clause
  * @param {object} options - Options passed to the bulk api.
  * @param {boolean} [options.autoFetch=false] - Fetch next records if available.
- * @param {function} callback - A callback to execute once the record is retrieved
  * @returns {Operation}
  */
-export function query(qs, options, callback = s => s) {
+export function query(qs, options) {
   return async state => {
     let done = false;
     let qResult = null;
@@ -497,11 +482,7 @@ export function query(qs, options, callback = s => s) {
       'Results retrieved and pushed to position [0] of the references array.'
     );
 
-    const nextState = {
-      ...state,
-      references: [result, ...state.references],
-    };
-    return callback(nextState);
+    return util.prepareNextState(state, ...result);
   };
 }
 
diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
index 62e47e85b..af5e5ec99 100644
--- a/packages/salesforce/src/Utils.js
+++ b/packages/salesforce/src/Utils.js
@@ -1,4 +1,6 @@
 import jsforce from 'jsforce';
+import { composeNextState } from '@openfn/language-common';
+import { result } from 'lodash';
 
 function getConnection(state, options) {
   const { apiVersion } = state.configuration;
@@ -134,3 +136,13 @@ export async function pollJobResult(conn, job, pollInterval, pollTimeout) {
     attempt++;
   }
 }
+
+export const prepareNextState = (state, fullResult) => {
+  const { records, ...result } = fullResult;
+
+  // console.log(fullResult);
+  return {
+    ...composeNextState(state, records || result),
+    result,
+  };
+};

From 7dff5f047a158e55c0eb47b0529af72cf0f77c2e Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Thu, 18 Jul 2024 21:23:22 +0300
Subject: [PATCH 05/22] remove lodash import

---
 packages/salesforce/src/Utils.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
index af5e5ec99..4b544af4b 100644
--- a/packages/salesforce/src/Utils.js
+++ b/packages/salesforce/src/Utils.js
@@ -1,6 +1,5 @@
 import jsforce from 'jsforce';
 import { composeNextState } from '@openfn/language-common';
-import { result } from 'lodash';
 
 function getConnection(state, options) {
   const { apiVersion } = state.configuration;

From 4b887e2f3e30c6c0d025db21f8f1302f3bcc40f5 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Fri, 19 Jul 2024 18:31:47 +0300
Subject: [PATCH 06/22] update prepNextState function

---
 packages/salesforce/src/Utils.js | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
index 4b544af4b..d12303ab4 100644
--- a/packages/salesforce/src/Utils.js
+++ b/packages/salesforce/src/Utils.js
@@ -136,12 +136,9 @@ export async function pollJobResult(conn, job, pollInterval, pollTimeout) {
   }
 }
 
-export const prepareNextState = (state, fullResult) => {
-  const { records, ...result } = fullResult;
-
-  // console.log(fullResult);
+export const prepareNextState = (state, result, data) => {
   return {
-    ...composeNextState(state, records || result),
+    ...composeNextState(state, data ?? result),
     result,
   };
 };

From 2ce39e68236f8212bb556ee3f49416b332cc0d59 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Fri, 19 Jul 2024 18:32:06 +0300
Subject: [PATCH 07/22] add get and post function

and improvements on other functions
---
 packages/salesforce/ast.json       |  87 +---------
 packages/salesforce/src/Adaptor.js | 256 +++++++++++++++--------------
 2 files changed, 143 insertions(+), 200 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index a1ba836d6..749454847 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -320,7 +320,7 @@
       "name": "destroy",
       "params": [
         "sObject",
-        "attrs",
+        "ids",
         "options"
       ],
       "docs": {
@@ -356,7 +356,7 @@
               "type": "NameExpression",
               "name": "object"
             },
-            "name": "attrs"
+            "name": "ids"
           },
           {
             "title": "param",
@@ -546,7 +546,7 @@
       "params": [
         "sObject",
         "externalId",
-        "attrs"
+        "records"
       ],
       "docs": {
         "description": "Create a new sObject record, or updates it if it already exists\nExternal ID field name must be specified in second argument.",
@@ -622,7 +622,7 @@
                 }
               ]
             },
-            "name": "attrs"
+            "name": "records"
           },
           {
             "title": "magic",
@@ -644,7 +644,7 @@
       "name": "update",
       "params": [
         "sObject",
-        "attrs"
+        "records"
       ],
       "docs": {
         "description": "Update an sObject record or records.",
@@ -703,7 +703,7 @@
                 }
               ]
             },
-            "name": "attrs"
+            "name": "records"
           },
           {
             "title": "returns",
@@ -759,8 +759,7 @@
       "name": "retrieve",
       "params": [
         "sObject",
-        "id",
-        "callback"
+        "id"
       ],
       "docs": {
         "description": "Retrieves a Salesforce sObject(s).",
@@ -797,15 +796,6 @@
             },
             "name": "id"
           },
-          {
-            "title": "param",
-            "description": "A callback to execute once the record is retrieved",
-            "type": {
-              "type": "NameExpression",
-              "name": "function"
-            },
-            "name": "callback"
-          },
           {
             "title": "returns",
             "description": null,
@@ -817,69 +807,6 @@
         ]
       },
       "valid": true
-    },
-    {
-      "name": "relationship",
-      "params": [
-        "relationshipName",
-        "externalId",
-        "dataSource"
-      ],
-      "docs": {
-        "description": "Adds a lookup relation or 'dome insert' to a record.",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "Data Sourced Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", dataSource(\"path\"))\nFixed Value:\n relationship(\"relationship_name__r\", \"externalID on related object\", \"hello world\")"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
-          {
-            "title": "param",
-            "description": "`__r` relationship field on the record.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "relationshipName"
-          },
-          {
-            "title": "param",
-            "description": "Salesforce ExternalID field.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "externalId"
-          },
-          {
-            "title": "param",
-            "description": "resolvable source.",
-            "type": {
-              "type": "NameExpression",
-              "name": "string"
-            },
-            "name": "dataSource"
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "object"
-            }
-          }
-        ]
-      },
-      "valid": true
     }
   ],
   "exports": [],
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index e835875e7..52dd1a752 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -13,13 +13,11 @@
 
 import {
   execute as commonExecute,
-  expandReferences,
-  composeNextState,
   field,
   chunk,
 } from '@openfn/language-common';
 
-import { expandReferences as newExpandReferences } from '@openfn/language-common/util';
+import { expandReferences } from '@openfn/language-common/util';
 import * as util from './Utils';
 
 import flatten from 'lodash/flatten';
@@ -98,7 +96,7 @@ export function execute(...operations) {
  * @param {integer} [options.pollTimeout=240000] - Polling timeout in milliseconds.
  * @returns {Operation}
  */
-export function bulk(sObject, operation, records, options) {
+export function bulk(sObject, operation, records, options = {}) {
   return state => {
     const { connection } = state;
 
@@ -107,7 +105,7 @@ export function bulk(sObject, operation, records, options) {
       resolvedOperation,
       resolvedRecords,
       resolvedOptions,
-    ] = newExpandReferences(state, sObject, operation, options, records);
+    ] = expandReferences(state, sObject, operation, records, options);
 
     const {
       failOnError = false,
@@ -132,9 +130,6 @@ export function bulk(sObject, operation, records, options) {
       chunkedBatches.map(
         chunkedBatch =>
           new Promise((resolve, reject) => {
-            const timeout = pollTimeout || 240000;
-            const interval = pollInterval || 6000;
-
             console.info(
               `Creating bulk ${resolvedOperation} job for ${resolvedSObject} with ${chunkedBatch.length} records`
             );
@@ -164,7 +159,7 @@ export function bulk(sObject, operation, records, options) {
                 console.info(batchInfo);
                 const batchId = batchInfo.id;
                 var batch = job.batch(batchId);
-                batch.poll(interval, timeout);
+                batch.poll(pollInterval, pollTimeout);
               })
               .then(async res => {
                 await job.close();
@@ -191,16 +186,11 @@ export function bulk(sObject, operation, records, options) {
       )
     ).then(arrayOfResults => {
       console.log('Merging results arrays.');
-      const merged = [].concat.apply([], arrayOfResults);
+      const merged = arrayOfResults.flat();
       return util.prepareNextState(state, merged);
-      return { ...state, references: [merged, ...state.references] };
     });
   };
 }
-// const defaultOptions = {
-//   pollTimeout: 90000, // in ms
-//   pollInterval: 3000, // in ms
-// };
 /**
  * Execute an SOQL Bulk Query.
  * This function uses bulk query to efficiently query large data sets and reduce the number of API requests.
@@ -223,18 +213,15 @@ export function bulk(sObject, operation, records, options) {
  * @param {integer} [options.pollInterval=3000] - Polling interval in milliseconds.
  * @returns {Operation}
  */
-export function bulkQuery(qs, options) {
+export function bulkQuery(qs, options = {}) {
   return async state => {
     const { connection } = state;
-    const [
-      resolvedQs,
-      resolvedOptions = { pollTimeout: 90000, pollInterval: 3000 },
-    ] = newExpandReferences(state, qs, options);
+    const [resolvedQs, resolvedOptions] = expandReferences(state, qs, options);
 
     if (parseFloat(connection.version) < 47.0)
       throw new Error('bulkQuery requires API version 47.0 and later');
 
-    const { pollTimeout, pollInterval } = resolvedOptions;
+    const { pollTimeout = 90000, pollInterval = 3000 } = resolvedOptions;
 
     console.log(`Executing query: ${resolvedQs}`);
 
@@ -276,7 +263,7 @@ export function bulkQuery(qs, options) {
 export function create(sObject, records) {
   return state => {
     let { connection } = state;
-    const [resolvedSObject, resolvedRecords] = newExpandReferences(
+    const [resolvedSObject, resolvedRecords] = expandReferences(
       state,
       sObject,
       records
@@ -305,7 +292,7 @@ export function describe(sObject) {
   return state => {
     const { connection } = state;
 
-    const [resolvedSObject] = newExpandReferences(state, sObject);
+    const [resolvedSObject] = expandReferences(state, sObject);
 
     return connection
       .sobject(resolvedSObject)
@@ -329,20 +316,25 @@ export function describe(sObject) {
  * ], { failOnError: true })
  * @function
  * @param {string} sObject - API name of the sObject.
- * @param {object} attrs - Array of IDs of records to delete.
+ * @param {object} ids - Array of IDs of records to delete.
  * @param {object} options - Options for the destroy delete operation.
  * @returns {Operation}
  */
-export function destroy(sObject, attrs, options) {
+export function destroy(sObject, ids, options = {}) {
   return state => {
     const { connection } = state;
-    const finalAttrs = expandReferences(attrs)(state);
-    const { failOnError } = options;
+    const [resolvedSObject, resolvedIds, resolvedOptions] = expandReferences(
+      state,
+      sObject,
+      ids,
+      options
+    );
+    const { failOnError = false } = resolvedOptions;
     console.info(`Deleting ${sObject} records`);
 
     return connection
-      .sobject(sObject)
-      .del(finalAttrs)
+      .sobject(resolvedSObject)
+      .del(resolvedIds)
       .then(function (result) {
         const successes = result.filter(r => r.success);
         console.log(
@@ -351,15 +343,13 @@ export function destroy(sObject, attrs, options) {
         );
 
         const failures = result.filter(r => !r.success);
-        console.log('Failed to delete: ', JSON.stringify(failures, null, 2));
+        if (failures.length > 0)
+          console.log('Failed to delete: ', JSON.stringify(failures, null, 2));
 
         if (failOnError && result.some(r => !r.success))
           throw 'Some deletes failed; exiting with failure code.';
 
-        return {
-          ...state,
-          references: [result, ...state.references],
-        };
+        return util.prepareNextState(state, result);
       });
   };
 }
@@ -379,10 +369,41 @@ export function describeAll() {
     return connection.describeGlobal().then(result => {
       const { sobjects } = result;
       console.log(`Retrieved ${sobjects.length} sObjects`);
-      return util.prepareNextState(state, { records: sobjects, ...result });
+      return util.prepareNextState(state, result, sobjects);
     });
   };
 }
+/**
+ * Send a GET HTTP request using connected session information.
+ * @example
+ * get('/actions/custom/flow/POC_OpenFN_Test_Flow');
+ * @param {string} url - Relative to request from
+ * @param {object} options - Request query parameters
+ * @returns {Operation}
+ */
+export function get(path, options = {}) {
+  return async state => {
+    const { connection } = state;
+    const [resolvedPath, resolvedOptions] = expandReferences(
+      state,
+      path,
+      options
+    );
+    const { headers, ...query } = resolvedOptions;
+    console.log(`GET: ${resolvedPath}`);
+    const requestOptions = {
+      url: resolvedPath,
+      method: 'GET',
+      query,
+      headers: { 'content-type': 'application/json', ...headers },
+    };
+
+    const result = await connection.request(requestOptions);
+
+    return util.prepareNextState(state, result);
+  };
+}
+
 /**
  * Alias for "create(sObject, attrs)".
  * @public
@@ -399,8 +420,43 @@ export function insert(sObject, records) {
   return create(sObject, records);
 }
 
-export function post(url, data, options) {
-  return state;
+/**
+ * Send a POST HTTP request using connected session information.
+ *
+ * @example
+ * post('/actions/custom/flow/POC_OpenFN_Test_Flow', { inputs: [{}] });
+ * @param {string} url - Relative to request from
+ * @param {object} data - A JSON Object request body
+ * @param {object} options - Request options
+ * @param {object} [options.headers] - Object of request headers
+ * @param {object} [options.query] - A JSON Object request body
+ * @returns {Operation}
+ */
+export function post(path, data, options = {}) {
+  return async state => {
+    const { connection } = state;
+    const [resolvedPath, resolvedData, resolvedOptions] = expandReferences(
+      state,
+      path,
+      data,
+      options
+    );
+    const { query, headers } = resolvedOptions;
+
+    console.log(`POST: ${resolvedPath}`);
+
+    const requestOptions = {
+      url: resolvedPath,
+      method: 'POST',
+      query,
+      headers: { 'content-type': 'application/json', ...headers },
+      body: JSON.stringify(resolvedData),
+    };
+
+    const result = await connection.request(requestOptions);
+
+    return util.prepareNextState(state, result);
+  };
 }
 
 /**
@@ -420,19 +476,15 @@ export function post(url, data, options) {
  * @param {boolean} [options.autoFetch=false] - Fetch next records if available.
  * @returns {Operation}
  */
-export function query(qs, options) {
+export function query(qs, options = {}) {
   return async state => {
     let done = false;
     let qResult = null;
     let result = [];
 
     const { connection } = state;
-    const [resolvedQs, resolvedOptions] = newExpandReferences(
-      state,
-      qs,
-      options
-    );
-    const { autoFetch } = { ...{ autoFetch: false }, ...resolvedOptions };
+    const [resolvedQs, resolvedOptions] = expandReferences(state, qs, options);
+    const { autoFetch = false } = resolvedOptions;
 
     console.log(`Executing query: ${resolvedQs}`);
     try {
@@ -478,11 +530,7 @@ export function query(qs, options) {
       console.log('No records found.');
     }
 
-    console.log(
-      'Results retrieved and pushed to position [0] of the references array.'
-    );
-
-    return util.prepareNextState(state, ...result);
+    return util.prepareNextState(state, result, result?.records);
   };
 }
 
@@ -502,29 +550,28 @@ export function query(qs, options) {
  * @magic sObject - $.children[?(!@.meta.system)].name
  * @param {string} externalId - The external ID of the sObject.
  * @magic externalId - $.children[?(@.name=="{{args.sObject}}")].children[?(@.meta.externalId)].name
- * @param {(object|object[])} attrs - Field attributes for the new object.
+ * @param {(object|object[])} records - Field attributes for the new object.
  * @magic attrs - $.children[?(@.name=="{{args.sObject}}")].children[?(!@.meta.externalId)]
  * @returns {Operation}
  */
-export function upsert(sObject, externalId, attrs) {
+export function upsert(sObject, externalId, records) {
   return state => {
     const { connection } = state;
-    const finalAttrs = expandReferences(attrs)(state);
+    const [resolvedSObject, resolvedExternalId, resolvedRecords] =
+      expandReferences(state, sObject, externalId, records);
     console.info(
-      `Upserting ${sObject} with externalId`,
-      externalId,
+      `Upserting ${resolvedSObject} with externalId`,
+      resolvedExternalId,
       ':',
-      finalAttrs
+      resolvedRecords
     );
 
     return connection
-      .upsert(sObject, finalAttrs, externalId)
-      .then(function (recordResult) {
-        console.log('Result : ' + JSON.stringify(recordResult));
-        return {
-          ...state,
-          references: [recordResult, ...state.references],
-        };
+      .upsert(resolvedSObject, resolvedRecords, resolvedExternalId)
+      .then(function (result) {
+        console.log('Result : ' + JSON.stringify(result));
+
+        return util.prepareNextState(state, result);
       });
   };
 }
@@ -544,22 +591,25 @@ export function upsert(sObject, externalId, attrs) {
  * ]);
  * @function
  * @param {string} sObject - API name of the sObject.
- * @param {(object|object[])} attrs - Field attributes for the new object.
+ * @param {(object|object[])} records - Field attributes for the new object.
  * @returns {Operation}
  */
-export function update(sObject, attrs) {
+export function update(sObject, records) {
   return state => {
     let { connection } = state;
-    const finalAttrs = expandReferences(attrs)(state);
-    console.info(`Updating ${sObject}`, finalAttrs);
-
-    return connection.update(sObject, finalAttrs).then(function (recordResult) {
-      console.log('Result : ' + JSON.stringify(recordResult));
-      return {
-        ...state,
-        references: [recordResult, ...state.references],
-      };
-    });
+    const [resolvedSObject, resolvedRecords] = expandReferences(
+      state,
+      sObject,
+      records
+    );
+    console.info(`Updating ${resolvedSObject}`, resolvedRecords);
+
+    return connection
+      .update(resolvedSObject, resolvedRecords)
+      .then(function (result) {
+        console.log('Result : ' + JSON.stringify(result));
+        return util.prepareNextState(state, result);
+      });
   };
 }
 
@@ -587,20 +637,18 @@ export function toUTF8(input) {
  *   method: 'POST',
  *   json: { inputs: [{}] },
  * });
- * @param {string} url - Relative or absolute URL to request from
+ * @param {string} url - Relative to request from
  * @param {object} options - Request options
  * @param {string} [options.method=GET] - HTTP method to use. Defaults to GET
  * @param {object} [options.headers] - Object of request headers
  * @param {object} [options.json] - A JSON Object request body
  * @param {string} [options.body] - HTTP body (in POST/PUT/PATCH methods)
- * @param {function} callback - A callback to execute once the request is complete
  * @returns {Operation}
  */
-
-export function request(path, options, callback = s => s) {
+export function request(path, options = {}) {
   return async state => {
     const { connection } = state;
-    const [resolvedPath, resolvedOptions] = newExpandReferences(
+    const [resolvedPath, resolvedOptions] = expandReferences(
       state,
       path,
       options
@@ -618,9 +666,7 @@ export function request(path, options, callback = s => s) {
 
     const result = await connection.request(requestOptions);
 
-    const nextState = composeNextState(state, result);
-
-    return callback(nextState);
+    return util.prepareNextState(state, result);
   };
 }
 
@@ -632,56 +678,26 @@ export function request(path, options, callback = s => s) {
  * @function
  * @param {string} sObject - The sObject to retrieve
  * @param {string} id - The id of the record
- * @param {function} callback - A callback to execute once the record is retrieved
  * @returns {Operation}
  */
-export function retrieve(sObject, id, callback) {
+export function retrieve(sObject, id) {
   return state => {
     const { connection } = state;
 
-    const finalId = expandReferences(id)(state);
+    const [resolvedSObject, resolvedId] = expandReferences(state, sObject, id);
 
+    console.log(
+      `Retrieving data for sObject '${resolvedSObject}' with Id '${resolvedId}'`
+    );
     return connection
-      .sobject(sObject)
-      .retrieve(finalId)
+      .sobject(resolvedSObject)
+      .retrieve(resolvedId)
       .then(result => {
-        return {
-          ...state,
-          references: [result, ...state.references],
-        };
-      })
-      .then(state => {
-        if (callback) {
-          return callback(state);
-        }
-        return state;
+        return util.prepareNextState(state, result);
       });
   };
 }
 
-/**
- * Adds a lookup relation or 'dome insert' to a record.
- * @public
- * @example
- * Data Sourced Value:
- *  relationship("relationship_name__r", "externalID on related object", dataSource("path"))
- * Fixed Value:
- *  relationship("relationship_name__r", "externalID on related object", "hello world")
- * @function
- * @param {string} relationshipName - `__r` relationship field on the record.
- * @param {string} externalId - Salesforce ExternalID field.
- * @param {string} dataSource - resolvable source.
- * @returns {object}
- */
-export function relationship(relationshipName, externalId, dataSource) {
-  return field(relationshipName, state => {
-    if (typeof dataSource == 'function') {
-      return { [externalId]: dataSource(state) };
-    }
-    return { [externalId]: dataSource };
-  });
-}
-
 export {
   alterState,
   arrayToString,

From 8d3a36fbe001f3e29b9d640e617c55c0f5ab9323 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Fri, 19 Jul 2024 18:32:52 +0300
Subject: [PATCH 08/22] fix broken test

---
 packages/salesforce/test/Adaptor.test.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/salesforce/test/Adaptor.test.js b/packages/salesforce/test/Adaptor.test.js
index 9816ae07a..37cb89bb2 100644
--- a/packages/salesforce/test/Adaptor.test.js
+++ b/packages/salesforce/test/Adaptor.test.js
@@ -26,7 +26,7 @@ describe('Adaptor', () => {
         .then(state => {
           expect(spy.args[0]).to.eql([sObject, fields]);
           expect(spy.called).to.eql(true);
-          expect(state.references[0]).to.eql({ Id: 10 });
+          expect(state.data).to.eql({ Id: 10 });
         })
         .then(done)
         .catch(done);
@@ -56,7 +56,7 @@ describe('Adaptor', () => {
         .then(state => {
           expect(spy.args[0]).to.eql([sObject, fields, externalId]);
           expect(spy.called).to.eql(true);
-          expect(state.references[0]).to.eql({ Id: 10 });
+          expect(state.data).to.eql({ Id: 10 });
         })
         .then(done)
         .catch(done);

From 7d5af16134c6fa46cb7da21a81ffac8283208a1d Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Mon, 22 Jul 2024 12:23:54 +0300
Subject: [PATCH 09/22] remove field, was unused

---
 packages/salesforce/src/Adaptor.js | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 52dd1a752..2a3472cbb 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -11,11 +11,7 @@
  * @ignore
  */
 
-import {
-  execute as commonExecute,
-  field,
-  chunk,
-} from '@openfn/language-common';
+import { execute as commonExecute, chunk } from '@openfn/language-common';
 
 import { expandReferences } from '@openfn/language-common/util';
 import * as util from './Utils';

From e324469dd90e8d944b348a0654fe7c2c2d248f5b Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Mon, 22 Jul 2024 12:24:09 +0300
Subject: [PATCH 10/22] add changeset

---
 .changeset/gentle-oranges-crash.md | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 .changeset/gentle-oranges-crash.md

diff --git a/.changeset/gentle-oranges-crash.md b/.changeset/gentle-oranges-crash.md
new file mode 100644
index 000000000..6c649b564
--- /dev/null
+++ b/.changeset/gentle-oranges-crash.md
@@ -0,0 +1,24 @@
+---
+'@openfn/language-salesforce': major
+---
+
+New API design for salesforce, including adding composeNextState and removing
+old code.
+
+### Major Changes
+
+- Remove axios dependency
+- Remove old/unused functions. `relationship`, `upsertIf`, `createIf`,
+  `reference`, `steps`, `beta`
+- Standardize state mutation in all operation
+- Change `bulk` signature to `bulk(operation, sObjectName, records, options)`
+- Remove callback support
+
+### Minor Changes
+
+- Create a get and post function for all http requests in salesforce
+
+### Patch Changes
+
+- Change `cleanupState` to `removeConnection` and tagged it as private function
+- Rename `attrs` to `records`

From 9902bc141ed1a60a9250910f76018224fe744b5e Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Mon, 22 Jul 2024 12:27:53 +0300
Subject: [PATCH 11/22] use resolvedOptions

---
 packages/salesforce/src/Adaptor.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 2a3472cbb..457862e7f 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -133,7 +133,7 @@ export function bulk(sObject, operation, records, options = {}) {
             const job = connection.bulk.createJob(
               resolvedSObject,
               resolvedOperation,
-              options
+              resolvedOptions
             );
 
             job.on('error', err => reject(err));
@@ -166,8 +166,8 @@ export function bulk(sObject, operation, records, options = {}) {
                   });
 
                 errors.forEach(err => {
-                  err[`${options.extIdField}`] =
-                    chunkedBatch[err.position - 1][options.extIdField];
+                  err[`${resolvedOptions.extIdField}`] =
+                    chunkedBatch[err.position - 1][resolvedOptions.extIdField];
                 });
 
                 if (failOnError && errors.length > 0) {

From 8c8fcf896c9e57a710f5bfef537427c22a996dac Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 09:09:27 +0300
Subject: [PATCH 12/22] remove describeAll() and improvements

---
 packages/salesforce/ast.json       |  81 ++++++--------
 packages/salesforce/src/Adaptor.js | 166 +++++++++++++----------------
 2 files changed, 107 insertions(+), 140 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index 749454847..c3528635b 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -3,7 +3,7 @@
     {
       "name": "bulk",
       "params": [
-        "sObject",
+        "sObjectName",
         "operation",
         "records",
         "options"
@@ -38,7 +38,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
@@ -79,6 +79,19 @@
             },
             "name": "options.extIdField"
           },
+          {
+            "title": "param",
+            "description": "Skipping bulk operation if no records.",
+            "type": {
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "boolean"
+              }
+            },
+            "name": "options.allowNoOp",
+            "default": "false"
+          },
           {
             "title": "param",
             "description": "Fail the operation on error.",
@@ -217,7 +230,7 @@
     {
       "name": "create",
       "params": [
-        "sObject",
+        "sObjectName",
         "records"
       ],
       "docs": {
@@ -250,7 +263,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
@@ -276,7 +289,7 @@
     {
       "name": "describe",
       "params": [
-        "sObject"
+        "sObjectName"
       ],
       "docs": {
         "description": "Prints an sObject metadata and pushes the result to state.references",
@@ -302,7 +315,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "returns",
@@ -319,7 +332,7 @@
     {
       "name": "destroy",
       "params": [
-        "sObject",
+        "sObjectName",
         "ids",
         "options"
       ],
@@ -333,7 +346,7 @@
           },
           {
             "title": "example",
-            "description": "destroy('obj_name', [\n '0060n00000JQWHYAA5',\n '0090n00000JQEWHYAA5\n], { failOnError: true })"
+            "description": "destroy('obj_name', [\n '0060n00000JQWHYAA5',\n '0090n00000JQEWHYAA5'\n], { failOnError: true })"
           },
           {
             "title": "function",
@@ -347,7 +360,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
@@ -379,46 +392,14 @@
       },
       "valid": true
     },
-    {
-      "name": "describeAll",
-      "params": [],
-      "docs": {
-        "description": "Prints the total number of all available sObjects and pushes the result to `state.references`.",
-        "tags": [
-          {
-            "title": "public",
-            "description": null,
-            "type": null
-          },
-          {
-            "title": "example",
-            "description": "describeAll()"
-          },
-          {
-            "title": "function",
-            "description": null,
-            "name": null
-          },
-          {
-            "title": "returns",
-            "description": null,
-            "type": {
-              "type": "NameExpression",
-              "name": "Operation"
-            }
-          }
-        ]
-      },
-      "valid": true
-    },
     {
       "name": "insert",
       "params": [
-        "sObject",
+        "sObjectName",
         "records"
       ],
       "docs": {
-        "description": "Alias for \"create(sObject, attrs)\".",
+        "description": "Alias for \"create(sObjectName, attrs)\".",
         "tags": [
           {
             "title": "public",
@@ -447,7 +428,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
@@ -544,7 +525,7 @@
     {
       "name": "upsert",
       "params": [
-        "sObject",
+        "sObjectName",
         "externalId",
         "records"
       ],
@@ -578,7 +559,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "magic",
@@ -643,7 +624,7 @@
     {
       "name": "update",
       "params": [
-        "sObject",
+        "sObjectName",
         "records"
       ],
       "docs": {
@@ -676,7 +657,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
@@ -758,7 +739,7 @@
     {
       "name": "retrieve",
       "params": [
-        "sObject",
+        "sObjectName",
         "id"
       ],
       "docs": {
@@ -785,7 +766,7 @@
               "type": "NameExpression",
               "name": "string"
             },
-            "name": "sObject"
+            "name": "sObjectName"
           },
           {
             "title": "param",
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 457862e7f..ecb738a11 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -31,23 +31,18 @@ const loadAnyAscii = state =>
 /**
  * Executes an operation.
  * @function
+ * @private
  * @param {Operation} operations - Operations
  * @returns {State}
  */
 export function execute(...operations) {
   const initialState = {
-    logger: {
-      info: console.info.bind(console),
-      debug: console.log.bind(console),
-    },
     references: [],
     data: null,
     configuration: {},
   };
 
   return state => {
-    // Note: we no longer need `steps` anymore since `commonExecute`
-    // takes each operation as an argument.
     return commonExecute(
       loadAnyAscii,
       util.createConnection,
@@ -82,26 +77,27 @@ export function execute(...operations) {
  *   { extIdField: "vera__Result_UID__c" }
  * );
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @param {string} operation - The bulk operation to be performed.Eg "insert" | "update" | "upsert"
  * @param {array} records - an array of records, or a function which returns an array.
  * @param {object} options - Options passed to the bulk api.
  * @param {string} [options.extIdField] - External id field.
+ * @param {boolean} [options.allowNoOp=false] - Skipping bulk operation if no records.
  * @param {boolean} [options.failOnError=false] - Fail the operation on error.
  * @param {integer} [options.pollInterval=6000] - Polling interval in milliseconds.
  * @param {integer} [options.pollTimeout=240000] - Polling timeout in milliseconds.
  * @returns {Operation}
  */
-export function bulk(sObject, operation, records, options = {}) {
+export function bulk(sObjectName, operation, records, options = {}) {
   return state => {
     const { connection } = state;
 
     const [
-      resolvedSObject,
+      resolvedSObjectName,
       resolvedOperation,
       resolvedRecords,
       resolvedOptions,
-    ] = expandReferences(state, sObject, operation, records, options);
+    ] = expandReferences(state, sObjectName, operation, records, options);
 
     const {
       failOnError = false,
@@ -112,7 +108,7 @@ export function bulk(sObject, operation, records, options = {}) {
 
     if (allowNoOp && resolvedRecords.length === 0) {
       console.info(
-        `No items in ${resolvedSObject} array. Skipping bulk ${resolvedOperation} operation.`
+        `No items in ${resolvedSObjectName} array. Skipping bulk ${resolvedOperation} operation.`
       );
       return state;
     }
@@ -127,11 +123,11 @@ export function bulk(sObject, operation, records, options = {}) {
         chunkedBatch =>
           new Promise((resolve, reject) => {
             console.info(
-              `Creating bulk ${resolvedOperation} job for ${resolvedSObject} with ${chunkedBatch.length} records`
+              `Creating bulk ${resolvedOperation} job for ${resolvedSObjectName} with ${chunkedBatch.length} records`
             );
 
             const job = connection.bulk.createJob(
-              resolvedSObject,
+              resolvedSObjectName,
               resolvedOperation,
               resolvedOptions
             );
@@ -180,10 +176,9 @@ export function bulk(sObject, operation, records, options = {}) {
               });
           })
       )
-    ).then(arrayOfResults => {
+    ).then(results => {
       console.log('Merging results arrays.');
-      const merged = arrayOfResults.flat();
-      return util.prepareNextState(state, merged);
+      return util.prepareNextState(state, results.flat());
     });
   };
 }
@@ -252,22 +247,22 @@ export function bulkQuery(qs, options = {}) {
  * @example <caption> Multiple records creation</caption>
  * create("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @param {object} records - Field attributes for the new record.
  * @returns {Operation}
  */
-export function create(sObject, records) {
+export function create(sObjectName, records) {
   return state => {
     let { connection } = state;
-    const [resolvedSObject, resolvedRecords] = expandReferences(
+    const [resolvedSObjectName, resolvedRecords] = expandReferences(
       state,
-      sObject,
+      sObjectName,
       records
     );
-    console.info(`Creating ${resolvedSObject}`, resolvedRecords);
+    console.info(`Creating ${resolvedSObjectName}`, resolvedRecords);
 
     return connection
-      .create(resolvedSObject, resolvedRecords)
+      .create(resolvedSObjectName, resolvedRecords)
       .then(recordResult => {
         console.log('Result : ' + JSON.stringify(recordResult));
         return util.prepareNextState(state, recordResult);
@@ -281,24 +276,30 @@ export function create(sObject, records) {
  * @example
  * describe('obj_name')
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @returns {Operation}
  */
-export function describe(sObject) {
+export function describe(sObjectName) {
   return state => {
     const { connection } = state;
 
-    const [resolvedSObject] = expandReferences(state, sObject);
+    const [resolvedSObjectName] = expandReferences(state, sObjectName);
 
-    return connection
-      .sobject(resolvedSObject)
-      .describe()
-      .then(result => {
-        console.log('Label : ' + result.label);
-        console.log('Num of Fields : ' + result.fields.length);
+    return resolvedSObjectName
+      ? connection
+          .sobject(resolvedSObjectName)
+          .describe()
+          .then(result => {
+            console.log('Label : ' + result.label);
+            console.log('Num of Fields : ' + result.fields.length);
 
-        return util.prepareNextState(state, result);
-      });
+            return util.prepareNextState(state, result);
+          })
+      : connection.describeGlobal().then(result => {
+          const { sobjects } = result;
+          console.log(`Retrieved ${sobjects.length} sObjects`);
+          return util.prepareNextState(state, result, sobjects);
+        });
   };
 }
 
@@ -308,72 +309,53 @@ export function describe(sObject) {
  * @example
  * destroy('obj_name', [
  *  '0060n00000JQWHYAA5',
- *  '0090n00000JQEWHYAA5
+ *  '0090n00000JQEWHYAA5'
  * ], { failOnError: true })
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @param {object} ids - Array of IDs of records to delete.
  * @param {object} options - Options for the destroy delete operation.
  * @returns {Operation}
  */
-export function destroy(sObject, ids, options = {}) {
+export function destroy(sObjectName, ids, options = {}) {
   return state => {
     const { connection } = state;
-    const [resolvedSObject, resolvedIds, resolvedOptions] = expandReferences(
-      state,
-      sObject,
-      ids,
-      options
-    );
+    const [resolvedSObjectName, resolvedIds, resolvedOptions] =
+      expandReferences(state, sObjectName, ids, options);
+
     const { failOnError = false } = resolvedOptions;
+
     console.info(`Deleting ${sObject} records`);
 
     return connection
-      .sobject(resolvedSObject)
+      .sobject(resolvedSObjectName)
       .del(resolvedIds)
       .then(function (result) {
         const successes = result.filter(r => r.success);
+        const failures = result.filter(r => !r.success);
+
         console.log(
           'Sucessfully deleted: ',
           JSON.stringify(successes, null, 2)
         );
 
-        const failures = result.filter(r => !r.success);
-        if (failures.length > 0)
+        if (failures.length > 0) {
           console.log('Failed to delete: ', JSON.stringify(failures, null, 2));
 
-        if (failOnError && result.some(r => !r.success))
-          throw 'Some deletes failed; exiting with failure code.';
+          if (failOnError)
+            throw 'Some deletes failed; exiting with failure code.';
+        }
 
         return util.prepareNextState(state, result);
       });
   };
 }
 
-/**
- * Prints the total number of all available sObjects and pushes the result to `state.references`.
- * @public
- * @example
- * describeAll()
- * @function
- * @returns {Operation}
- */
-export function describeAll() {
-  return state => {
-    const { connection } = state;
-
-    return connection.describeGlobal().then(result => {
-      const { sobjects } = result;
-      console.log(`Retrieved ${sobjects.length} sObjects`);
-      return util.prepareNextState(state, result, sobjects);
-    });
-  };
-}
 /**
  * Send a GET HTTP request using connected session information.
  * @example
  * get('/actions/custom/flow/POC_OpenFN_Test_Flow');
- * @param {string} url - Relative to request from
+ * @param {string} path - The Salesforce API endpoint, Relative to request from
  * @param {object} options - Request query parameters
  * @returns {Operation}
  */
@@ -401,19 +383,19 @@ export function get(path, options = {}) {
 }
 
 /**
- * Alias for "create(sObject, attrs)".
+ * Alias for "create(sObjectName, attrs)".
  * @public
  * @example <caption> Single record creation</caption>
  * insert("Account", { Name: "My Account #1" });
  * @example <caption> Multiple records creation</caption>
  * insert("Account",[{ Name: "My Account #1" }, { Name: "My Account #2" }]);
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @param {object} records - Field attributes for the new record.
  * @returns {Operation}
  */
-export function insert(sObject, records) {
-  return create(sObject, records);
+export function insert(sObjectName, records) {
+  return create(sObjectName, records);
 }
 
 /**
@@ -421,7 +403,7 @@ export function insert(sObject, records) {
  *
  * @example
  * post('/actions/custom/flow/POC_OpenFN_Test_Flow', { inputs: [{}] });
- * @param {string} url - Relative to request from
+ * @param {string} path - The Salesforce API endpoint, Relative to request from
  * @param {object} data - A JSON Object request body
  * @param {object} options - Request options
  * @param {object} [options.headers] - Object of request headers
@@ -542,7 +524,7 @@ export function query(qs, options = {}) {
  *   { Name: "Record #2", ExtId__c : 'ID-0000002' },
  * ]);
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @magic sObject - $.children[?(!@.meta.system)].name
  * @param {string} externalId - The external ID of the sObject.
  * @magic externalId - $.children[?(@.name=="{{args.sObject}}")].children[?(@.meta.externalId)].name
@@ -550,20 +532,20 @@ export function query(qs, options = {}) {
  * @magic attrs - $.children[?(@.name=="{{args.sObject}}")].children[?(!@.meta.externalId)]
  * @returns {Operation}
  */
-export function upsert(sObject, externalId, records) {
+export function upsert(sObjectName, externalId, records) {
   return state => {
     const { connection } = state;
-    const [resolvedSObject, resolvedExternalId, resolvedRecords] =
-      expandReferences(state, sObject, externalId, records);
+    const [resolvedSObjectName, resolvedExternalId, resolvedRecords] =
+      expandReferences(state, sObjectName, externalId, records);
     console.info(
-      `Upserting ${resolvedSObject} with externalId`,
+      `Upserting ${resolvedSObjectName} with externalId`,
       resolvedExternalId,
       ':',
       resolvedRecords
     );
 
     return connection
-      .upsert(resolvedSObject, resolvedRecords, resolvedExternalId)
+      .upsert(resolvedSObjectName, resolvedRecords, resolvedExternalId)
       .then(function (result) {
         console.log('Result : ' + JSON.stringify(result));
 
@@ -586,22 +568,22 @@ export function upsert(sObject, externalId, records) {
  *   { Id: "0010500000fxbcvAAA", Name: "Updated Account #2" },
  * ]);
  * @function
- * @param {string} sObject - API name of the sObject.
+ * @param {string} sObjectName - API name of the sObject.
  * @param {(object|object[])} records - Field attributes for the new object.
  * @returns {Operation}
  */
-export function update(sObject, records) {
+export function update(sObjectName, records) {
   return state => {
     let { connection } = state;
-    const [resolvedSObject, resolvedRecords] = expandReferences(
+    const [resolvedSObjectName, resolvedRecords] = expandReferences(
       state,
-      sObject,
+      sObjectName,
       records
     );
-    console.info(`Updating ${resolvedSObject}`, resolvedRecords);
+    console.info(`Updating ${resolvedSObjectName}`, resolvedRecords);
 
     return connection
-      .update(resolvedSObject, resolvedRecords)
+      .update(resolvedSObjectName, resolvedRecords)
       .then(function (result) {
         console.log('Result : ' + JSON.stringify(result));
         return util.prepareNextState(state, result);
@@ -634,7 +616,7 @@ export function toUTF8(input) {
  *   json: { inputs: [{}] },
  * });
  * @param {string} url - Relative to request from
- * @param {object} options - Request options
+ * @param {object} options - Query parameters and headers
  * @param {string} [options.method=GET] - HTTP method to use. Defaults to GET
  * @param {object} [options.headers] - Object of request headers
  * @param {object} [options.json] - A JSON Object request body
@@ -672,21 +654,25 @@ export function request(path, options = {}) {
  * @example
  * retrieve('ContentVersion', '0684K0000020Au7QAE/VersionData');
  * @function
- * @param {string} sObject - The sObject to retrieve
+ * @param {string} sObjectName - The sObject to retrieve
  * @param {string} id - The id of the record
  * @returns {Operation}
  */
-export function retrieve(sObject, id) {
+export function retrieve(sObjectName, id) {
   return state => {
     const { connection } = state;
 
-    const [resolvedSObject, resolvedId] = expandReferences(state, sObject, id);
+    const [resolvedSObjectName, resolvedId] = expandReferences(
+      state,
+      sObjectName,
+      id
+    );
 
     console.log(
-      `Retrieving data for sObject '${resolvedSObject}' with Id '${resolvedId}'`
+      `Retrieving data for sObject '${resolvedSObjectName}' with Id '${resolvedId}'`
     );
     return connection
-      .sobject(resolvedSObject)
+      .sobject(resolvedSObjectName)
       .retrieve(resolvedId)
       .then(result => {
         return util.prepareNextState(state, result);

From 3d8ffaffac812d81a6b3653684181bbbbb0a419a Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 09:09:53 +0300
Subject: [PATCH 13/22] wip: add migration guide

---
 .changeset/gentle-oranges-crash.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.changeset/gentle-oranges-crash.md b/.changeset/gentle-oranges-crash.md
index 6c649b564..c9427c43b 100644
--- a/.changeset/gentle-oranges-crash.md
+++ b/.changeset/gentle-oranges-crash.md
@@ -9,7 +9,7 @@ old code.
 
 - Remove axios dependency
 - Remove old/unused functions. `relationship`, `upsertIf`, `createIf`,
-  `reference`, `steps`, `beta`
+  `reference`, `steps`, `beta`, `describeAll()`
 - Standardize state mutation in all operation
 - Change `bulk` signature to `bulk(operation, sObjectName, records, options)`
 - Remove callback support
@@ -22,3 +22,5 @@ old code.
 
 - Change `cleanupState` to `removeConnection` and tagged it as private function
 - Rename `attrs` to `records`
+
+### Migration Guide

From 660f86f89851880c7815d7b3baaa22b1c5e068d1 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 11:10:25 +0300
Subject: [PATCH 14/22] improve docs for describe

---
 packages/salesforce/ast.json       | 19 ++++++++++++++-----
 packages/salesforce/src/Adaptor.js | 11 +++++++----
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index c3528635b..164f6d0a1 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -292,7 +292,7 @@
         "sObjectName"
       ],
       "docs": {
-        "description": "Prints an sObject metadata and pushes the result to state.references",
+        "description": "Fetches and prints metadata for an sObject and pushes the result to `state.data`.\nIf `sObjectName` is not specified, it will print the total number of all available sObjects and push the result to `state.data`.",
         "tags": [
           {
             "title": "public",
@@ -301,7 +301,13 @@
           },
           {
             "title": "example",
-            "description": "describe('obj_name')"
+            "description": "describe()",
+            "caption": "Fetch metadata for all available sObjects"
+          },
+          {
+            "title": "example",
+            "description": "describe('Account')",
+            "caption": "Fetch metadata for Account sObject"
           },
           {
             "title": "function",
@@ -310,10 +316,13 @@
           },
           {
             "title": "param",
-            "description": "API name of the sObject.",
+            "description": "The API name of the sObject. If omitted, fetches metadata for all sObjects.",
             "type": {
-              "type": "NameExpression",
-              "name": "string"
+              "type": "OptionalType",
+              "expression": {
+                "type": "NameExpression",
+                "name": "string"
+              }
             },
             "name": "sObjectName"
           },
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index ecb738a11..6d3f44395 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -271,12 +271,15 @@ export function create(sObjectName, records) {
 }
 
 /**
- * Prints an sObject metadata and pushes the result to state.references
+ * Fetches and prints metadata for an sObject and pushes the result to `state.data`.
+ * If `sObjectName` is not specified, it will print the total number of all available sObjects and push the result to `state.data`.
  * @public
- * @example
- * describe('obj_name')
+ * @example <caption>Fetch metadata for all available sObjects</caption>
+ * describe()
+ * @example <caption>Fetch metadata for Account sObject</caption>
+ * describe('Account')
  * @function
- * @param {string} sObjectName - API name of the sObject.
+ * @param {string} [sObjectName] - The API name of the sObject. If omitted, fetches metadata for all sObjects.
  * @returns {Operation}
  */
 export function describe(sObjectName) {

From efe7f8d1c0290e0a2e858bcb9e86fa0c360cd762 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 11:10:42 +0300
Subject: [PATCH 15/22] breakdown changeset into individual updates

---
 .changeset/gentle-oranges-crash.md | 21 ++++++++++++---------
 .changeset/lucky-chicken-notice.md |  7 +++++++
 .changeset/wise-trees-carry.md     |  6 ++++++
 3 files changed, 25 insertions(+), 9 deletions(-)
 create mode 100644 .changeset/lucky-chicken-notice.md
 create mode 100644 .changeset/wise-trees-carry.md

diff --git a/.changeset/gentle-oranges-crash.md b/.changeset/gentle-oranges-crash.md
index c9427c43b..73c670856 100644
--- a/.changeset/gentle-oranges-crash.md
+++ b/.changeset/gentle-oranges-crash.md
@@ -5,8 +5,6 @@
 New API design for salesforce, including adding composeNextState and removing
 old code.
 
-### Major Changes
-
 - Remove axios dependency
 - Remove old/unused functions. `relationship`, `upsertIf`, `createIf`,
   `reference`, `steps`, `beta`, `describeAll()`
@@ -14,13 +12,18 @@ old code.
 - Change `bulk` signature to `bulk(operation, sObjectName, records, options)`
 - Remove callback support
 
-### Minor Changes
-
-- Create a get and post function for all http requests in salesforce
-
-### Patch Changes
+### Migration Guide
 
-- Change `cleanupState` to `removeConnection` and tagged it as private function
-- Rename `attrs` to `records`
+- Use `describe()` instead of `describeAll()`.
+- Use `fnIf(true, upsert())` instead of `upsertIf()`
+- Use `fnIf(true, create())` instead of `createIf()`
+- Use `bulk(operation, sObjectName, records, options )` instead of
+  `bulk(operation, sObject, options, records)`
 
 ### Migration Guide
+
+- Replace `describeAll()` with `describe()`.
+- Replace `upsertIf()` with `fnIf(true, upsert())`.
+- Replace `createIf()` with `fnIf(true, create())`.
+- Replace `bulk(operation, sObject, options, records)` with
+  `bulk(operation, sObjectName, records, options)`.
diff --git a/.changeset/lucky-chicken-notice.md b/.changeset/lucky-chicken-notice.md
new file mode 100644
index 000000000..0adc131bb
--- /dev/null
+++ b/.changeset/lucky-chicken-notice.md
@@ -0,0 +1,7 @@
+---
+'@openfn/language-salesforce': minor
+---
+
+- Create a `get()` and `post()` function for all http requests in salesforce
+  tions, records)`
+- Update `describe()` to fetch all available sObjects metadata
diff --git a/.changeset/wise-trees-carry.md b/.changeset/wise-trees-carry.md
new file mode 100644
index 000000000..8bac0ed1f
--- /dev/null
+++ b/.changeset/wise-trees-carry.md
@@ -0,0 +1,6 @@
+---
+'@openfn/language-salesforce': patch
+---
+
+- Change `cleanupState` to `removeConnection` and tagged it as private function
+- Rename `attrs` to `records`

From c38736fe2ea2eddc28972a58755be1205085a641 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 11:39:54 +0300
Subject: [PATCH 16/22] remove duplicate

---
 .changeset/gentle-oranges-crash.md | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.changeset/gentle-oranges-crash.md b/.changeset/gentle-oranges-crash.md
index 73c670856..ba7d83753 100644
--- a/.changeset/gentle-oranges-crash.md
+++ b/.changeset/gentle-oranges-crash.md
@@ -14,14 +14,6 @@ old code.
 
 ### Migration Guide
 
-- Use `describe()` instead of `describeAll()`.
-- Use `fnIf(true, upsert())` instead of `upsertIf()`
-- Use `fnIf(true, create())` instead of `createIf()`
-- Use `bulk(operation, sObjectName, records, options )` instead of
-  `bulk(operation, sObject, options, records)`
-
-### Migration Guide
-
 - Replace `describeAll()` with `describe()`.
 - Replace `upsertIf()` with `fnIf(true, upsert())`.
 - Replace `createIf()` with `fnIf(true, create())`.

From 45fe867390c0fcb0cb67bc793f38b3961cff7e5b Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 12:41:09 +0300
Subject: [PATCH 17/22] update request options

---
 packages/salesforce/src/Adaptor.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 6d3f44395..fa1e289cf 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -619,10 +619,10 @@ export function toUTF8(input) {
  *   json: { inputs: [{}] },
  * });
  * @param {string} url - Relative to request from
- * @param {object} options - Query parameters and headers
+ * @param {object} options - The options for the request.
  * @param {string} [options.method=GET] - HTTP method to use. Defaults to GET
  * @param {object} [options.headers] - Object of request headers
- * @param {object} [options.json] - A JSON Object request body
+ * @param {object} [options.json] - A JSON object to send as the request body.
  * @param {string} [options.body] - HTTP body (in POST/PUT/PATCH methods)
  * @returns {Operation}
  */

From 5687603d0fd6badd04211e8d4807736007ddca23 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 13:53:33 +0300
Subject: [PATCH 18/22] update get docs

---
 packages/salesforce/src/Adaptor.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index fa1e289cf..868f148c7 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -359,7 +359,7 @@ export function destroy(sObjectName, ids, options = {}) {
  * @example
  * get('/actions/custom/flow/POC_OpenFN_Test_Flow');
  * @param {string} path - The Salesforce API endpoint, Relative to request from
- * @param {object} options - Request query parameters
+ * @param {object} options - Request query parameters and headers
  * @returns {Operation}
  */
 export function get(path, options = {}) {

From b4dc8af1a567263e96afeb6ebcf2777e967637e0 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 14:07:53 +0300
Subject: [PATCH 19/22] improve next state composition

---
 packages/salesforce/src/Adaptor.js | 2 +-
 packages/salesforce/src/Utils.js   | 7 +------
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 868f148c7..af7d8fc89 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -301,7 +301,7 @@ export function describe(sObjectName) {
       : connection.describeGlobal().then(result => {
           const { sobjects } = result;
           console.log(`Retrieved ${sobjects.length} sObjects`);
-          return util.prepareNextState(state, result, sobjects);
+          return util.prepareNextState(state, result);
         });
   };
 }
diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
index d12303ab4..1cb7e5ef8 100644
--- a/packages/salesforce/src/Utils.js
+++ b/packages/salesforce/src/Utils.js
@@ -136,9 +136,4 @@ export async function pollJobResult(conn, job, pollInterval, pollTimeout) {
   }
 }
 
-export const prepareNextState = (state, result, data) => {
-  return {
-    ...composeNextState(state, data ?? result),
-    result,
-  };
-};
+export const prepareNextState = (state, data) => composeNextState(state, data);

From d1c178e5073e123cc55fa2ed9ccf82fa4b6a8fc2 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 17:24:35 +0300
Subject: [PATCH 20/22] fix lookup test

---
 packages/salesforce/ast.json                     | 4 ++--
 packages/salesforce/src/Adaptor.js               | 4 ++--
 packages/salesforce/test/metadata/lookup.test.js | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/packages/salesforce/ast.json b/packages/salesforce/ast.json
index 164f6d0a1..d33f7fbe3 100644
--- a/packages/salesforce/ast.json
+++ b/packages/salesforce/ast.json
@@ -572,7 +572,7 @@
           },
           {
             "title": "magic",
-            "description": "sObject - $.children[?(!@.meta.system)].name"
+            "description": "sObjectName - $.children[?(!@.meta.system)].name"
           },
           {
             "title": "param",
@@ -616,7 +616,7 @@
           },
           {
             "title": "magic",
-            "description": "attrs - $.children[?(@.name==\"{{args.sObject}}\")].children[?(!@.meta.externalId)]"
+            "description": "records - $.children[?(@.name==\"{{args.sObject}}\")].children[?(!@.meta.externalId)]"
           },
           {
             "title": "returns",
diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index af7d8fc89..8fbf5a4ca 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -528,11 +528,11 @@ export function query(qs, options = {}) {
  * ]);
  * @function
  * @param {string} sObjectName - API name of the sObject.
- * @magic sObject - $.children[?(!@.meta.system)].name
+ * @magic sObjectName - $.children[?(!@.meta.system)].name
  * @param {string} externalId - The external ID of the sObject.
  * @magic externalId - $.children[?(@.name=="{{args.sObject}}")].children[?(@.meta.externalId)].name
  * @param {(object|object[])} records - Field attributes for the new object.
- * @magic attrs - $.children[?(@.name=="{{args.sObject}}")].children[?(!@.meta.externalId)]
+ * @magic records - $.children[?(@.name=="{{args.sObject}}")].children[?(!@.meta.externalId)]
  * @returns {Operation}
  */
 export function upsert(sObjectName, externalId, records) {
diff --git a/packages/salesforce/test/metadata/lookup.test.js b/packages/salesforce/test/metadata/lookup.test.js
index 7e8a96d47..f14993b65 100644
--- a/packages/salesforce/test/metadata/lookup.test.js
+++ b/packages/salesforce/test/metadata/lookup.test.js
@@ -15,7 +15,7 @@ describe('Salesforce lookup tests', async () => {
   // Unit tests of each query against the cached metadata
   describe('upsert', () => {
     it('sObject: should list non-system sObject names', () => {
-      const results = jp.query(data, queries.upsert.sObject);
+      const results = jp.query(data, queries.upsert.sObjectName);
       // Note that there are two sobjects in the model - this should correctly just return 1
       expect(results).to.have.lengthOf(1);
       expect(results).to.include('vera__Beneficiary__c');

From e479fe0febc7af144479cf4372e9dbb77b7d6c55 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 17:32:55 +0300
Subject: [PATCH 21/22] eslint fix

---
 packages/salesforce/src/Adaptor.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index 8fbf5a4ca..f2bc4f251 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -328,7 +328,7 @@ export function destroy(sObjectName, ids, options = {}) {
 
     const { failOnError = false } = resolvedOptions;
 
-    console.info(`Deleting ${sObject} records`);
+    console.info(`Deleting ${resolvedSObjectName} records`);
 
     return connection
       .sobject(resolvedSObjectName)

From 98c1f8326575e2ae07a1464baf53ab21cebfcc33 Mon Sep 17 00:00:00 2001
From: Emmanuel Evance <mtuchidev@gmail.com>
Date: Tue, 23 Jul 2024 18:36:31 +0300
Subject: [PATCH 22/22] remove prepareNextState

---
 packages/salesforce/src/Adaptor.js | 32 +++++++++++++++++-------------
 packages/salesforce/src/Utils.js   |  3 ---
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/packages/salesforce/src/Adaptor.js b/packages/salesforce/src/Adaptor.js
index f2bc4f251..531bc6c7e 100644
--- a/packages/salesforce/src/Adaptor.js
+++ b/packages/salesforce/src/Adaptor.js
@@ -11,7 +11,11 @@
  * @ignore
  */
 
-import { execute as commonExecute, chunk } from '@openfn/language-common';
+import {
+  execute as commonExecute,
+  chunk,
+  composeNextState,
+} from '@openfn/language-common';
 
 import { expandReferences } from '@openfn/language-common/util';
 import * as util from './Utils';
@@ -178,7 +182,7 @@ export function bulk(sObjectName, operation, records, options = {}) {
       )
     ).then(results => {
       console.log('Merging results arrays.');
-      return util.prepareNextState(state, results.flat());
+      return composeNextState(state, results.flat());
     });
   };
 }
@@ -235,7 +239,7 @@ export function bulkQuery(qs, options = {}) {
       pollTimeout
     );
 
-    return util.prepareNextState(state, result);
+    return composeNextState(state, result);
   };
 }
 
@@ -265,7 +269,7 @@ export function create(sObjectName, records) {
       .create(resolvedSObjectName, resolvedRecords)
       .then(recordResult => {
         console.log('Result : ' + JSON.stringify(recordResult));
-        return util.prepareNextState(state, recordResult);
+        return composeNextState(state, recordResult);
       });
   };
 }
@@ -296,12 +300,12 @@ export function describe(sObjectName) {
             console.log('Label : ' + result.label);
             console.log('Num of Fields : ' + result.fields.length);
 
-            return util.prepareNextState(state, result);
+            return composeNextState(state, result);
           })
       : connection.describeGlobal().then(result => {
           const { sobjects } = result;
           console.log(`Retrieved ${sobjects.length} sObjects`);
-          return util.prepareNextState(state, result);
+          return composeNextState(state, result);
         });
   };
 }
@@ -349,7 +353,7 @@ export function destroy(sObjectName, ids, options = {}) {
             throw 'Some deletes failed; exiting with failure code.';
         }
 
-        return util.prepareNextState(state, result);
+        return composeNextState(state, result);
       });
   };
 }
@@ -381,7 +385,7 @@ export function get(path, options = {}) {
 
     const result = await connection.request(requestOptions);
 
-    return util.prepareNextState(state, result);
+    return composeNextState(state, result);
   };
 }
 
@@ -436,7 +440,7 @@ export function post(path, data, options = {}) {
 
     const result = await connection.request(requestOptions);
 
-    return util.prepareNextState(state, result);
+    return composeNextState(state, result);
   };
 }
 
@@ -511,7 +515,7 @@ export function query(qs, options = {}) {
       console.log('No records found.');
     }
 
-    return util.prepareNextState(state, result, result?.records);
+    return composeNextState(state, result, result?.records);
   };
 }
 
@@ -552,7 +556,7 @@ export function upsert(sObjectName, externalId, records) {
       .then(function (result) {
         console.log('Result : ' + JSON.stringify(result));
 
-        return util.prepareNextState(state, result);
+        return composeNextState(state, result);
       });
   };
 }
@@ -589,7 +593,7 @@ export function update(sObjectName, records) {
       .update(resolvedSObjectName, resolvedRecords)
       .then(function (result) {
         console.log('Result : ' + JSON.stringify(result));
-        return util.prepareNextState(state, result);
+        return composeNextState(state, result);
       });
   };
 }
@@ -647,7 +651,7 @@ export function request(path, options = {}) {
 
     const result = await connection.request(requestOptions);
 
-    return util.prepareNextState(state, result);
+    return composeNextState(state, result);
   };
 }
 
@@ -678,7 +682,7 @@ export function retrieve(sObjectName, id) {
       .sobject(resolvedSObjectName)
       .retrieve(resolvedId)
       .then(result => {
-        return util.prepareNextState(state, result);
+        return composeNextState(state, result);
       });
   };
 }
diff --git a/packages/salesforce/src/Utils.js b/packages/salesforce/src/Utils.js
index 1cb7e5ef8..62e47e85b 100644
--- a/packages/salesforce/src/Utils.js
+++ b/packages/salesforce/src/Utils.js
@@ -1,5 +1,4 @@
 import jsforce from 'jsforce';
-import { composeNextState } from '@openfn/language-common';
 
 function getConnection(state, options) {
   const { apiVersion } = state.configuration;
@@ -135,5 +134,3 @@ export async function pollJobResult(conn, job, pollInterval, pollTimeout) {
     attempt++;
   }
 }
-
-export const prepareNextState = (state, data) => composeNextState(state, data);