salesforce Fix autoFetch in query

.changeset/ten-crews-complain.md

@@ -0,0 +1,8 @@
+---
+'@openfn/language-salesforce': major
+---
+
+Fix `autoFetch` in `query()` function
+
+- All records are merged in a single`records` array
+- Results are pushed to `[0]` in `state.references`

packages/salesforce/src/Adaptor.js

@@ -164,19 +164,28 @@ export function retrieve(sObject, id, callback) {
  * @param {function} callback - A callback to execute once the record is retrieved
  * @returns {Operation}
  */
-export function query(qs, options, callback = s => s) {
+export function query(qs, options = {}, callback = s => s) {
   return async state => {
-    let done = false;
     let qResult = null;
-    let result = [];
+    let result = {
+      done: true,
+      totalSize: 0,
+      records: [],
+    };
 
     const { connection } = state;
     const [resolvedQs, resolvedOptions] = newExpandReferences(
       state,
       qs,
       options
     );
-    const { autoFetch } = { ...{ autoFetch: false }, ...resolvedOptions };
+
+    const { autoFetch: autoFetchOption, autofetch: autoFetchOptionAlias } =
+      resolvedOptions;
+    const autoFetch = autoFetchOption || autoFetchOptionAlias || false;
+    if (autoFetch) {
+      console.log('AutoFetch is enabled');
+    }
 
     console.log(`Executing query: ${resolvedQs}`);
     try {
@@ -187,38 +196,37 @@ export function query(qs, options, callback = s => s) {
       throw err;
     }
 
-    if (qResult.totalSize > 0) {
-      console.log('Total records', qResult.totalSize);
-
-      while (!done) {
-        result.push(qResult);
-
-        if (qResult.done) {
-          done = true;
-        } else if (autoFetch) {
-          console.log(
-            'Fetched records so far',
-            result.map(ref => ref.records).flat().length
-          );
-          console.log('Fetching next records...');
-          try {
-            qResult = await connection.request({ url: qResult.nextRecordsUrl });
-          } catch (err) {
-            const { message, errorCode } = err;
-            console.log(`Error ${errorCode}: ${message}`);
-            throw err;
-          }
-        } else {
-          done = true;
+    const processRecords = async qResult => {
+      const { done, totalSize, records, nextRecordsUrl } = qResult;
+
+      result.done = done;
+      result.totalSize = totalSize;
+      result.records.push(...records);
+
+      if (!done && !autoFetch && nextRecordsUrl) {
+        result.nextRecordsUrl = nextRecordsUrl;
+      }
+      if (!done && autoFetch) {
+        console.log('Fetched records so far:', result.records.length);
+        console.log('Fetching next records...');
+
+        try {
+          const newResult = await connection.request({ url: nextRecordsUrl });
+          await processRecords(newResult);
+        } catch (err) {
+          const { message, errorCode } = err;
+          console.error(`Error ${errorCode}: ${message}`);
+          throw err;
         }
       }
+    };
 
-      console.log(
-        'Done ✔ retrieved records',
-        result.map(ref => ref.records).flat().length
-      );
+    if (qResult.totalSize > 0) {
+      console.log('Total records:', qResult.totalSize);
+      await processRecords(qResult);
+      console.log('Done ✔ retrieved records:', result.records.length);
     } else {
-      result.push(qResult);
+      result = qResult;
       console.log('No records found.');
     }
 

packages/salesforce/test/Adaptor.test.js

@@ -8,6 +8,7 @@ import {
   upsertIf,
   toUTF8,
   execute,
+  query,
 } from '../src/Adaptor';
 
 const { expect } = chai;
@@ -194,4 +195,138 @@ describe('Adaptor', () => {
       expect(result).to.eql('Nhamaonha 6a Classe 2023-10-09');
     });
   });
+
+  describe('query', () => {
+    it('should properly pass the query string', async () => {
+      const fakeConnection = {
+        query: function () {
+          return Promise.resolve({
+            done: true,
+            totalSize: 1,
+            records: [{ Name: 'OpenFn' }],
+          });
+        },
+      };
+
+      let state = {
+        connection: fakeConnection,
+        references: [],
+      };
+      let spy = sinon.spy(fakeConnection, 'query');
+
+      query('select Name from Account')(state).then(state => {
+        expect(spy.calledWith('select Name from Account')).to.eql(true);
+      });
+    });
+    it('should properly pass the query string from state', async () => {
+      const fakeConnection = {
+        query: function () {
+          return Promise.resolve({
+            done: true,
+            totalSize: 1,
+            records: [{ Name: 'OpenFn' }],
+          });
+        },
+      };
+
+      let state = {
+        connection: fakeConnection,
+        references: [],
+        qs: 'select Name from Account',
+      };
+      let spy = sinon.spy(fakeConnection, 'query');
+
+      query(state => state.qs)(state).then(state => {
+        expect(spy.calledWith('select Name from Account')).to.eql(true);
+      });
+    });
+
+    it('should fetch up to 2000 records', done => {
+      const fakeConnection = {
+        query: function () {
+          return Promise.resolve({
+            done: true,
+            totalSize: 1,
+            records: [{ Name: 'OpenFn' }],
+          });
+        },
+      };
+      let state = { connection: fakeConnection, references: [] };
+      let spy = sinon.spy(fakeConnection, 'query');
+
+      query('select Name from Account')(state)
+        .then(state => {
+          expect(spy.called).to.eql(true);
+          expect(state.references[0]).to.eql({
+            done: true,
+            totalSize: 1,
+            records: [{ Name: 'OpenFn' }],
+          });
+        })
+        .then(done);
+    });
+    it('should fetch all page if autoFetch is true', done => {
+      const fakeConnection = {
+        query: function () {
+          return Promise.resolve({
+            done: false,
+            totalSize: 5713,
+            nextRecordsUrl:
+              '/services/data/v47.0/query/0r8yy3Dlrs3Ol9EACO-2000',
+            records: [{ Name: 'Open' }],
+          });
+        },
+        request: function () {
+          return Promise.resolve({
+            done: true,
+            totalSize: 5713,
+            records: [{ Name: 'Fn' }],
+          });
+        },
+      };
+      let state = { connection: fakeConnection, references: [] };
+      let spy = sinon.spy(fakeConnection, 'query');
+      let spyReq = sinon.spy(fakeConnection, 'request');
+
+      query('select Name from Account', { autoFetch: true })(state)
+        .then(state => {
+          expect(spy.called).to.eql(true);
+          expect(spyReq.called).to.eql(true);
+          expect(state.references[0]).to.eql({
+            done: true,
+            totalSize: 5713,
+            records: [{ Name: 'Open' }, { Name: 'Fn' }],
+          });
+        })
+        .then(done);
+    });
+    it('should not fetch another page if autofetch is false', done => {
+      const fakeConnection = {
+        query: function () {
+          return Promise.resolve({
+            done: false,
+            totalSize: 5713,
+            nextRecordsUrl:
+              '/services/data/v47.0/query/0r8yy3Dlrs3Ol9EACO-2000',
+            records: [{ Name: 'OpenFn' }],
+          });
+        },
+      };
+      let state = { connection: fakeConnection, references: [] };
+      let spy = sinon.spy(fakeConnection, 'query');
+
+      query('select Name from Account')(state)
+        .then(state => {
+          expect(spy.called).to.eql(true);
+          expect(state.references[0]).to.eql({
+            done: false,
+            totalSize: 5713,
+            nextRecordsUrl:
+              '/services/data/v47.0/query/0r8yy3Dlrs3Ol9EACO-2000',
+            records: [{ Name: 'OpenFn' }],
+          });
+        })
+        .then(done);
+    });
+  });
 });