Merge pull request #1417 from ashitsalesforce/master

reduce memory footproint for bulk and bulkv2 uploads
forcedotcom · Dec 25, 2024 · c13cdc4 · c13cdc4
2 parents 54c2a72 + ec5cb60
commit c13cdc4
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 15 deletions.
diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/DAOLoadVisitor.java
@@ -230,9 +230,8 @@ public boolean visit(TableRow row) throws OperationException, DataAccessObjectEx
             getLogger().error(errMsg, conve);
 
             conversionFailed(row, errMsg);
-            if (appConfig.getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)
-                    || (!appConfig.isBulkAPIEnabled() && !appConfig.isBulkV2APIEnabled())) {
-                // either bulk mode or cache bulk data uploaded from DAO
+            if (!appConfig.isBulkAPIEnabled() && !appConfig.isBulkV2APIEnabled()) {
+                // SOAP or REST API use daoRowList to process results of an upload request
                 this.daoRowList.add(row);
             }
             return false;
@@ -282,9 +281,7 @@ public void flushRemaining() throws OperationException, DataAccessObjectExceptio
 
     public void clearArrays() {
         // clear the arrays
-        if (!controller.getAppConfig().getBoolean(AppConfig.PROP_BULK_API_ENABLED)) {
-            daoRowList.clear();
-        }
+        daoRowList.clear();
         dynaArray.clear();
     }
 

diff --git a/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java b/src/main/java/com/salesforce/dataloader/action/visitor/bulk/BulkLoadVisitor.java
@@ -35,7 +35,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -476,9 +475,8 @@ private void getResults() throws AsyncApiException, OperationException, DataAcce
         	return;
         }
         DataReader dataReader = null;
-        if (!controller.getAppConfig().getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
-            dataReader = resetDAO();
-        }
+        dataReader = resetDAO();
+
         // create a map of batch infos by batch id. Each batchinfo has the final processing state of the batch
         final Map<String, BatchInfo> batchInfoMap = createBatchInfoMap();
 
@@ -517,11 +515,7 @@ private void processResults(final DataReader dataReader, final BatchInfo batch,
 
         final int totalRowsInDAOInCurrentBatch = lastDAORowForCurrentBatch - this.firstDAORowForCurrentBatch + 1;
         List<TableRow> rows;
-        if (controller.getAppConfig().getBoolean(AppConfig.PROP_PROCESS_BULK_CACHE_DATA_FROM_DAO)) {
-            rows = this.daoRowList.subList(firstDAORowForCurrentBatch, firstDAORowForCurrentBatch+totalRowsInDAOInCurrentBatch);
-        } else {
-            rows = dataReader.readTableRowList(totalRowsInDAOInCurrentBatch);
-        }
+        rows = dataReader.readTableRowList(totalRowsInDAOInCurrentBatch);
         if (batch.getState() == BatchStateEnum.Completed || batch.getNumberRecordsProcessed() > 0) {
             try {
                 processBatchResults(batch, errorMessage, batch.getState(), rows, this.firstDAORowForCurrentBatch);