configurable caching of global sobject describe and sobject fields

Make the caching of sobject and sobject field info configurable, thereby enabling refresh of sobject info as requested at https://ideas.salesforce.com/s/idea/a0B8W00000GdnjpUAB/refresh-the-data-structure-when-logged-in-data-loader
forcedotcom · Jan 15, 2024 · 7be0548 · 7be0548
1 parent ff1385b
commit 7be0548
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 62 deletions.
diff --git a/src/main/java/com/salesforce/dataloader/client/PartnerClient.java b/src/main/java/com/salesforce/dataloader/client/PartnerClient.java
@@ -223,10 +223,10 @@ public DescribeSObjectResult run(String entity) throws ConnectionException {
         }
     };
 
-    private DescribeGlobalResult entityTypes;
-    private final Map<String, DescribeRefObject> referenceDescribes = new HashMap<String, DescribeRefObject>();
-    private final Map<String, DescribeGlobalSObjectResult> describeGlobalResults = new HashMap<String, DescribeGlobalSObjectResult>();
-    private final Map<String, DescribeSObjectResult> entityDescribes = new HashMap<String, DescribeSObjectResult>();
+    private DescribeGlobalResult describeGlobalResults;
+    private final Map<String, DescribeRefObject> referenceEntitiesDescribesMap = new HashMap<String, DescribeRefObject>();
+    private final Map<String, DescribeGlobalSObjectResult> describeGlobalResultsMap = new HashMap<String, DescribeGlobalSObjectResult>();
+    private final Map<String, DescribeSObjectResult> entityFieldDescribesMap = new HashMap<String, DescribeSObjectResult>();
 
     private final boolean enableRetries;
     private final int maxRetries;
@@ -526,15 +526,31 @@ public PartnerConnection getClient() {
     }
 
     public Map<String, DescribeGlobalSObjectResult> getDescribeGlobalResults() {
-        return describeGlobalResults;
-    }
-
-    Map<String, DescribeSObjectResult> getEntityDescribeMap() {
-        return this.entityDescribes;
+        if (this.describeGlobalResults == null || !config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS)) {
+            this.describeGlobalResultsMap.clear();
+            try {
+                this.describeGlobalResults = runOperation(DESCRIBE_GLOBAL_OPERATION, null);
+            } catch (ConnectionException e) {
+                logger.error("Failed to get description of sobjects", e.getMessage());
+                return null;
+            }
+        }
+
+        if (this.describeGlobalResultsMap.isEmpty()) {
+            for (DescribeGlobalSObjectResult res : describeGlobalResults.getSobjects()) {
+                if (res != null) {
+                    if (res.getLabel().startsWith("__MISSING LABEL__")) {
+                        res.setLabel(res.getName());
+                    }
+                    this.describeGlobalResultsMap.put(res.getName(), res);
+                }
+            }
+        }
+        return describeGlobalResultsMap;
     }
 
-    DescribeGlobalResult getEntityTypes() {
-        return entityTypes;
+    private Map<String, DescribeSObjectResult> getCachedEntityDescribeMap() {
+        return this.entityFieldDescribesMap;
     }
 
     public DescribeSObjectResult getFieldTypes() {
@@ -547,7 +563,7 @@ public DescribeSObjectResult getFieldTypes() {
     }
 
     public Map<String, DescribeRefObject> getReferenceDescribes() {
-        return referenceDescribes;
+        return referenceEntitiesDescribesMap;
     }
 
     public LimitInfo getAPILimitInfo() {
@@ -749,43 +765,13 @@ private void retrySleep(String operationName, int retryNum) {
         }
     }
 
-    /**
-     * Gets the sObject describes for all entities
-     */
-    public boolean setEntityDescribes() throws ConnectionException {
-        setEntityTypes();
-        if (this.describeGlobalResults.isEmpty()) {
-            for (DescribeGlobalSObjectResult res : entityTypes.getSobjects()) {
-                if (res != null) {
-                    if (res.getLabel().startsWith("__MISSING LABEL__")) {
-                        res.setLabel(res.getName());
-                    }
-                    this.describeGlobalResults.put(res.getName(), res);
-                }
-            }
-        }
-
-        return true;
-    }
-
-    /**
-     * Gets the available objects from the global describe
-     */
-    private void setEntityTypes() throws ConnectionException {
-        if (this.entityTypes == null)
-            this.entityTypes = runOperation(DESCRIBE_GLOBAL_OPERATION, null);
-    }
-
     /**
      * Set the map of references to object external id info for current entity
      *
      * @throws ConnectionException
      */
     public void setFieldReferenceDescribes() throws ConnectionException {
-        referenceDescribes.clear();
-        if (getDescribeGlobalResults().isEmpty()) {
-            setEntityDescribes();
-        }
+        referenceEntitiesDescribesMap.clear();
         if (getFieldTypes() == null) {
             setFieldTypes();
         }
@@ -832,7 +818,7 @@ private void processParentObjectForLookupReferences(String parentObjectName, Fie
         }
         if (!parentIdLookupFieldMap.isEmpty()) {
             DescribeRefObject describeRelationship = new DescribeRefObject(parentObjectName, childObjectField, parentIdLookupFieldMap);
-            referenceDescribes.put(childObjectField.getRelationshipName(), describeRelationship);
+            referenceEntitiesDescribesMap.put(childObjectField.getRelationshipName(), describeRelationship);
         }
     }
 
@@ -918,11 +904,14 @@ private String getDefaultServer() {
      */
 
     public DescribeSObjectResult describeSObject(String entity) throws ConnectionException {
-        DescribeSObjectResult result = getEntityDescribeMap().get(entity);
+        DescribeSObjectResult result = null;
+        if (config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS)) {
+            result = getCachedEntityDescribeMap().get(entity);
+        }
         if (result == null) {
             result = runOperation(DESCRIBE_SOBJECT_OPERATION, entity);
             if (result != null) {
-                getEntityDescribeMap().put(result.getName(), result);
+                getCachedEntityDescribeMap().put(result.getName(), result);
             }
         }
         return result;

diff --git a/src/main/java/com/salesforce/dataloader/config/Config.java b/src/main/java/com/salesforce/dataloader/config/Config.java
@@ -212,7 +212,8 @@ public class Config {
     public static final String CSV_DELIMITER_OTHER_VALUE = "loader.csvOtherValue";
     public static final String CSV_DELIMITER_FOR_QUERY_RESULTS = "loader.query.delimiter";
     public static final String BUFFER_UNPROCESSED_BULK_QUERY_RESULTS = "loader.bufferUnprocessedBulkQueryResults";
-
+    public static final String CACHE_DESCRIBE_GLOBAL_RESULTS = "loader.cacheSObjectNamesAndFields";
+
     //Special Internal Configs
     public static final String SFDC_INTERNAL = "sfdcInternal"; //$NON-NLS-1$
     public static final String SFDC_INTERNAL_IS_SESSION_ID_LOGIN = "sfdcInternal.isSessionIdLogin"; //$NON-NLS-1$
@@ -606,6 +607,7 @@ private void setDefaults() {
         setDefaultValue(DAO_WRITE_POSTPROCESSOR_SCRIPT, "");
         setDefaultValue(LIMIT_OUTPUT_TO_QUERY_FIELDS, true);
         setDefaultValue(WIZARD_CLOSE_ON_FINISH, true);
+        setDefaultValue(CACHE_DESCRIBE_GLOBAL_RESULTS, true);
     }
 
     /**

diff --git a/src/main/java/com/salesforce/dataloader/controller/Controller.java b/src/main/java/com/salesforce/dataloader/controller/Controller.java
@@ -186,11 +186,6 @@ private boolean loginIfSessionExists(ClientBase<?> clientToLogin) {
     public boolean loginIfSessionExists() {
         return loginIfSessionExists(getClient());
     }
-
-    public boolean setEntityDescribes() throws ConnectionException {
-        validateSession();
-        return getPartnerClient().setEntityDescribes();
-    }
 
     public static void setAPIVersion(String apiVersionStr) {
         API_VERSION = apiVersionStr;

diff --git a/src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java b/src/main/java/com/salesforce/dataloader/ui/AdvancedSettingsDialog.java
@@ -103,6 +103,7 @@ public class AdvancedSettingsDialog extends BaseDialog {
     private Button buttonTruncateFields;
     private Button buttonFormatPhoneFields;
     private Button buttonKeepAccountTeam;
+    private Button buttonCacheDescribeGlobalResults;
     private Button buttonUseBulkApi;
     private Button buttonUseBulkV2Api;
     private Button buttonBulkApiSerialMode;
@@ -500,6 +501,15 @@ public void verifyText(VerifyEvent event) {
         data.widthHint = 5 * textSize.x;
         textQueryResultsDelimiterValue.setLayoutData(data);
 
+        Label labelCacheDescribeGlobalResults = new Label(restComp, SWT.RIGHT | SWT.WRAP);
+        labelCacheDescribeGlobalResults.setText(Labels.getString("AdvancedSettingsDialog.cacheDescribeGlobalResults"));
+        data = new GridData(GridData.HORIZONTAL_ALIGN_END);
+        labelCacheDescribeGlobalResults.setLayoutData(data);
+
+        boolean cacheDescribeGlobalResults = config.getBoolean(Config.CACHE_DESCRIBE_GLOBAL_RESULTS);
+        buttonCacheDescribeGlobalResults = new Button(restComp, SWT.CHECK);
+        buttonCacheDescribeGlobalResults.setSelection(cacheDescribeGlobalResults);
+
         // Keep Account team setting
         Label labelKeepAccountTeam = new Label(restComp, SWT.RIGHT | SWT.WRAP);
         labelKeepAccountTeam.setText(Labels.getString("AdvancedSettingsDialog.keepAccountTeam"));
@@ -520,7 +530,7 @@ public void widgetSelected(SelectionEvent e) {
         });
         buttonKeepAccountTeam.setToolTipText(Labels.getString("AdvancedSettingsDialog.keepAccountTeamHelp"));
         labelKeepAccountTeam.setToolTipText(Labels.getString("AdvancedSettingsDialog.keepAccountTeamHelp"));
-
+        
         // Enable Bulk API Setting
         Label labelUseBulkApi = new Label(restComp, SWT.RIGHT | SWT.WRAP);
         labelUseBulkApi.setText(Labels.getString("AdvancedSettingsDialog.useBulkApi")); //$NON-NLS-1$
@@ -886,6 +896,7 @@ public void widgetSelected(SelectionEvent event) {
                 config.setValue(Config.PROXY_USERNAME, textProxyUsername.getText());
                 config.setValue(Config.PROXY_NTLM_DOMAIN, textProxyNtlmDomain.getText());
                 config.setValue(Config.PROCESS_KEEP_ACCOUNT_TEAM, buttonKeepAccountTeam.getSelection());
+                config.setValue(Config.CACHE_DESCRIBE_GLOBAL_RESULTS, buttonCacheDescribeGlobalResults.getSelection());
                 config.setValue(Config.BULK_API_ENABLED, buttonUseBulkApi.getSelection());
                 config.setValue(Config.BULK_API_SERIAL_MODE, buttonBulkApiSerialMode.getSelection());
                 config.setValue(Config.BULK_API_ZIP_CONTENT, buttonBulkApiZipContent.getSelection());

diff --git a/src/main/java/com/salesforce/dataloader/ui/AuthenticationRunner.java b/src/main/java/com/salesforce/dataloader/ui/AuthenticationRunner.java
@@ -130,7 +130,7 @@ private void loginAsync(){
 	                return;
             	}
             }
-            if (controller.login() && controller.setEntityDescribes()) {
+            if (controller.login() && controller.getEntityDescribes() != null) {
                 messenger.accept(Labels.getString("SettingsPage.loginSuccessful"));
                 controller.saveConfig();
                 controller.updateLoaderWindowTitleAndCacheUserInfoForTheSession();

diff --git a/src/main/java/com/salesforce/dataloader/ui/SettingsPage.java b/src/main/java/com/salesforce/dataloader/ui/SettingsPage.java
@@ -141,8 +141,7 @@ public IWizardPage getNextPage() {
     }
 
     public static boolean isNeeded(Controller controller) {
-        return (!controller.loginIfSessionExists() || controller.getEntityDescribes() == null || controller
-                .getEntityDescribes().isEmpty());
+        return (!controller.isLoggedIn());
     }
 
     private void authenticationCompleted(Boolean success){

diff --git a/src/main/resources/labels.properties b/src/main/resources/labels.properties
@@ -117,6 +117,7 @@ AdvancedSettingsDialog.latestLoggingFile=Logging output file:
 AdvancedSettingsDialog.loggingConfigFile=Logging configuration file:
 AdvancedSettingsDialog.checkUploadDelimiterCheckbox=Specify delimiter(s) for upload operations.
 AdvancedSettingsDialog.closeWizardOnFinish=Close UI Wizard dialog when an operation is completed.
+AdvancedSettingsDialog.cacheDescribeGlobalResults=Cache Salesforce object and field information across operations
 
 InsertWizard.windowTitle=Load Inserts
 InsertWizard.confFirstLine=You have chosen to insert new records.  Click Yes to begin.

diff --git a/src/test/java/com/salesforce/dataloader/client/PartnerClientTest.java b/src/test/java/com/salesforce/dataloader/client/PartnerClientTest.java
@@ -202,17 +202,12 @@ public void testDisconnect() throws Exception {
     @Test
     public void testSetEntityDescribe() throws Exception{
         PartnerClient client = new PartnerClient(getController());
-        assertTrue(client.setEntityDescribes());
         assertNotNull(client.getDescribeGlobalResults());
-        assertEquals(client.getEntityTypes().getSobjects().length, client
-                .getDescribeGlobalResults().size());
     }
 
     @Test
     public void testDescribeSObjects() throws Exception {
         PartnerClient client = new PartnerClient(getController());
-        assertTrue(client.getEntityDescribeMap().isEmpty());
-        client.setEntityDescribes();
 
         int numDescribes = 0;
         for (String objectType : client.getDescribeGlobalResults().keySet()){
@@ -221,7 +216,6 @@ public void testDescribeSObjects() throws Exception {
         		numDescribes++;
                 assertNotNull(describeResult);
                 assertEquals(objectType, describeResult.getName());
-                assertEquals(numDescribes, client.getEntityDescribeMap().size());
         	} catch (Exception ex) {
         		if (ex.getMessage().contains("jsonNot")) {
         			System.out.println("PartnerClient.testDescribeSObjects: Unable to call describeSObject for " + objectType);