Merge pull request #103 from texei/fix-lookupoverride

Fix lookupoverride
texei · May 3, 2022 · 926c928 · 926c928
2 parents be116ca + fb87261
commit 926c928
Show file tree

Hide file tree

Showing 6 changed files with 232 additions and 155 deletions.
diff --git a/README.md b/README.md
@@ -21,8 +21,8 @@ Link the plugin: sfdx plugins:link .
 
 <!-- commands -->
 * [`sfdx texei:contractstatus:value:add -l <string> -a <string> [-s <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeicontractstatusvalueadd--l-string--a-string--s-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
-* [`sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataexport--d-string--o-string--p-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
-* [`sfdx texei:data:import -d <string> [-a] [-o] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataimport--d-string--a--o--u-string---apiversion-string---verbose---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
+* [`sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-a <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataexport--d-string--o-string--p-string--a-string--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
+* [`sfdx texei:data:import -d <string> [-a] [-o] [-p <string>] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataimport--d-string--a--o--p-string--u-string---apiversion-string---verbose---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
 * [`sfdx texei:data:plan:generate -d <string> -o <string> [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidataplangenerate--d-string--o-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
 * [`sfdx texei:debug:lwc:enable [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeidebuglwcenable--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
 * [`sfdx texei:org:contractfieldhistory:fix [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`](#sfdx-texeiorgcontractfieldhistoryfix--u-string---apiversion-string---json---loglevel-tracedebuginfowarnerrorfataltracedebuginfowarnerrorfatal)
@@ -77,16 +77,18 @@ EXAMPLE
 
 _See code: [src/commands/texei/contractstatus/value/add.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.14.4/src/commands/texei/contractstatus/value/add.ts)_
 
-## `sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`
+## `sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-a <string>] [-u <string>] [--apiversion <string>] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`
 
 export objects' data from org
 
 ```
 USAGE
-  $ sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-u <string>] [--apiversion <string>] [--json] 
-  [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]
+  $ sfdx texei:data:export -d <string> [-o <string>] [-p <string>] [-a <string>] [-u <string>] [--apiversion <string>] 
+  [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]
 
 OPTIONS
+  -a, --apitype=rest|bulk                                                           [default: rest] API Type to use
+
   -d, --outputdir=outputdir                                                         (required) directory where to store
                                                                                     files
 
@@ -113,14 +115,14 @@ EXAMPLES
 
 _See code: [src/commands/texei/data/export.ts](https://github.com/texei/texei-sfdx-plugin/blob/v1.14.4/src/commands/texei/data/export.ts)_
 
-## `sfdx texei:data:import -d <string> [-a] [-o] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`
+## `sfdx texei:data:import -d <string> [-a] [-o] [-p <string>] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]`
 
 import objects' data to org
 
 ```
 USAGE
-  $ sfdx texei:data:import -d <string> [-a] [-o] [-u <string>] [--apiversion <string>] [--verbose] [--json] [--loglevel 
-  trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]
+  $ sfdx texei:data:import -d <string> [-a] [-o] [-p <string>] [-u <string>] [--apiversion <string>] [--verbose] 
+  [--json] [--loglevel trace|debug|info|warn|error|fatal|TRACE|DEBUG|INFO|WARN|ERROR|FATAL]
 
 OPTIONS
   -a, --allornone                                                                   any failed records in a call cause
@@ -133,6 +135,8 @@ OPTIONS
   -o, --ignoreerrors                                                                errors are displayed as warnings
                                                                                     only and import will continue
 
+  -p, --dataplan=dataplan                                                           path to data plan file
+
   -u, --targetusername=targetusername                                               username or alias for the target
                                                                                     org; overrides default target org
 

diff --git a/messages/data-export.json b/messages/data-export.json
@@ -2,5 +2,6 @@
   "commandDescription": "export objects' data from org",
   "objectsFlagDescription": "comma-separated list of objects to export",
   "outputdirFlagDescription": "directory where to store files",
-  "dataPlanFlagDescription": "path to data plan file"
+  "dataPlanFlagDescription": "path to data plan file",
+  "apiTypeFlagDescription": "API Type to use"
 }
diff --git a/messages/data-import.json b/messages/data-import.json
@@ -3,5 +3,6 @@
     "inputFlagDescription": "directory with files to import",
     "allOrNoneFlagDescription": "any failed records in a call cause all changes for the call to be rolled back",
     "ignoreErrorsFlagDescription": "errors are displayed as warnings only and import will continue",
+    "dataPlanFlagDescription": "path to data plan file",
     "verbose": "verbose output of import result"
 }
diff --git a/src/commands/texei/data/DataPlan.d.ts b/src/commands/texei/data/DataPlan.d.ts
@@ -12,4 +12,5 @@ interface DataPlanSObject {
     externalId: string;
     excludedFields: Array<string>;
     lookupOverride?: {};
+    batchSize?: number;
 }
diff --git a/src/commands/texei/data/export.ts b/src/commands/texei/data/export.ts
@@ -1,10 +1,11 @@
 import { flags, SfdxCommand } from '@salesforce/command';
 import { Messages, SfdxError, Connection } from '@salesforce/core';
 import { AnyJson } from '@salesforce/ts-types';
-import { ExecuteOptions } from 'jsforce';
+import { Record, ExecuteOptions } from 'jsforce';
 import * as fs from 'fs';
 import * as path from 'path';
 const util = require("util");
+const csv = require("csvtojson");
 
 // Initialize Messages with the current plugin directory
 Messages.importMessagesDirectory(__dirname);
@@ -30,7 +31,8 @@ export default class Export extends SfdxCommand {
   protected static flagsConfig = {
     outputdir: flags.string({char: 'd', description: messages.getMessage('outputdirFlagDescription'), required: true}),
     objects: flags.string({char: 'o', description: messages.getMessage('objectsFlagDescription'), required: false}),
-    dataplan: flags.string({char: 'p', description: messages.getMessage('dataPlanFlagDescription'), required: false})
+    dataplan: flags.string({char: 'p', description: messages.getMessage('dataPlanFlagDescription'), required: false}),
+    apitype: flags.string({ char: 'a', description: messages.getMessage('apiTypeFlagDescription'), options: ['rest', 'bulk'], default: 'rest' })
   };
 
   // Comment this out if your command does not require an org username
@@ -197,13 +199,51 @@ export default class Export extends SfdxCommand {
                          FROM ${sobject.name}
                          ${sobject.filters ? 'WHERE '+sobject.filters : ''}
                          ${sobject.orderBy ? 'ORDER BY '+sobject.orderBy : ''}`;
-    // API Default limit is 10 000, just check if we need to extend it
-    const recordNumber:number = ((await conn.query(`Select count(Id) numberOfRecords from ${sobject.name}`)).records[0] as any).numberOfRecords;
-    let options:ExecuteOptions = {};
-    if (recordNumber > 10000) {
-      options.maxFetch = recordNumber;
+
+    let recordResults;
+
+    if (this.flags.apitype === 'bulk') {
+
+      const bulkQuery =  async (sObjectQuery: string) => new Promise<Array<Record>>(async (resolve, reject) => {
+        let retrievedRecords: Array<Record> = new Array<Record>();
+
+        conn.bulk.pollTimeout = 250000;
+
+        // Manually reading stream instead on using jsforce directly
+        // Because jsforce will return '75008.0' instead of 75008 for a number
+        const recordStream = conn.bulk.query(sObjectQuery);
+        const readStream = recordStream.stream();
+        const csvToJsonParser = csv({flatKeys: false, checkType: true});
+        readStream.pipe(csvToJsonParser);
+
+        csvToJsonParser.on("data", (data) => {
+          retrievedRecords.push(JSON.parse(data.toString('utf8')));
+        });
+
+        recordStream.on("error", (error) => {
+          reject(error);
+        });
+
+        csvToJsonParser.on("error", (error) => {
+          reject(error);
+        });
+
+        csvToJsonParser.on("done", async () => {
+          resolve(retrievedRecords);
+        });
+      });
+
+      recordResults = await bulkQuery(recordQuery);
+    }
+    else {
+      // API Default limit is 10 000, just check if we need to extend it
+      const recordNumber:number = ((await conn.query(`Select count(Id) numberOfRecords from ${sobject.name}`)).records[0] as any).numberOfRecords;
+      let options:ExecuteOptions = {};
+      if (recordNumber > 10000) {
+        options.maxFetch = recordNumber;
+      }
+      recordResults = (await conn.autoFetchQuery(recordQuery, options)).records;
     }
-    const recordResults = (await conn.autoFetchQuery(recordQuery, options)).records;
 
     // Replace Lookup Ids + Record Type Ids by references
     await this.cleanJsonRecord(sobject, sObjectLabel, recordResults, recordIdsMap, lookups, overriddenLookups, userFieldsReference);
@@ -233,6 +273,11 @@ export default class Export extends SfdxCommand {
 
     for (const record of records) {
 
+      if (record.attributes === undefined) {
+        // Not returned by bulk API
+        record.attributes = {};
+      }
+
       // Delete record url, useless to reimport somewhere else
       delete record.attributes.url;
 
@@ -267,7 +312,7 @@ export default class Export extends SfdxCommand {
         if (record[lookup]) {
           // If lookup isn't empty, remove useless information
           // Keeping "type" so we don't have to do a describe to know what is the related sObject at import
-          delete record[lookup].attributes.url;
+          delete record[lookup]?.attributes?.url;
         }
         else {
           // Remove empty lookup relationship field