fix: try to reduce sqlite cursor memory usage

amplitude · Jul 6, 2023 · 62a7734 · 62a7734
1 parent d950578
commit 62a7734
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 49 deletions.
diff --git a/src/main/java/com/amplitude/api/DatabaseHelper.java b/src/main/java/com/amplitude/api/DatabaseHelper.java
@@ -13,7 +13,6 @@
 import org.json.JSONObject;
 
 import java.io.File;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -340,26 +339,19 @@ synchronized List<JSONObject> getIdentifyInterceptors(
 
     protected synchronized List<JSONObject> getEventsFromTable(
                                     String table, long upToId, long limit) throws JSONException {
-        List<JSONObject> events = new LinkedList<JSONObject>();
+        List<Long> eventIds = new LinkedList<Long>();
         Cursor cursor = null;
         try {
             SQLiteDatabase db = getReadableDatabase();
             cursor = queryDb(
-                db, table, new String[] { ID_FIELD, EVENT_FIELD },
+                db, table, new String[] { ID_FIELD },
                 upToId >= 0 ? ID_FIELD + " <= " + upToId : null, null, null, null,
                 ID_FIELD + " ASC", limit >= 0 ? "" + limit : null
             );
 
             while (cursor.moveToNext()) {
                 long eventId = cursor.getLong(0);
-                String event = cursor.getString(1);
-                if (Utils.isEmptyString(event)) {
-                    continue;
-                }
-
-                JSONObject obj = new JSONObject(event);
-                obj.put("event_id", eventId);
-                events.add(obj);
+                eventIds.add(eventId);
             }
         } catch (SQLiteException e) {
             logger.e(TAG, String.format("getEvents from %s failed", table), e);
@@ -377,7 +369,55 @@ protected synchronized List<JSONObject> getEventsFromTable(
             }
             close();
         }
-        return events;
+
+        try {
+            List<JSONObject> events = new LinkedList<JSONObject>();
+            for (Long eventId : eventIds) {
+                JSONObject event = getEventFromTable(table, eventId);
+                if (event != null) {
+                    events.add(event);
+                }
+            }
+            return events;
+        } finally {
+            close();
+        }
+    }
+
+    protected synchronized JSONObject getEventFromTable(String table, long eventId) throws JSONException {
+        JSONObject event = null;
+        Cursor cursor = null;
+        try {
+            SQLiteDatabase db = getReadableDatabase();
+            cursor = queryDb(
+                db, table, new String[] { EVENT_FIELD },
+               ID_FIELD + " = " + eventId,
+                    null, null, null, null, null
+            );
+
+            if (cursor.moveToFirst()) {
+                String eventData = cursor.getString(0);
+                if (!Utils.isEmptyString(eventData)) {
+                    event = new JSONObject(eventData);
+                    event.put("event_id", eventId);
+                }
+            }
+        } catch (SQLiteException e) {
+            logger.e(TAG, String.format("getEvent from %s failed", table), e);
+            delete();
+        } catch (StackOverflowError e) {
+            logger.e(TAG, String.format("getEvent from %s failed", table), e);
+            delete();
+        } catch (IllegalStateException e) {  // put before Runtime since IllegalState extends
+            handleIfCursorRowTooLargeException(e);
+        } catch (RuntimeException e) {
+            convertIfCursorWindowException(e);
+        } finally {
+            if (cursor != null) {
+                cursor.close();
+            }
+        }
+        return event;
     }
 
     synchronized long getEventCount() {

diff --git a/src/main/java/com/amplitude/api/IdentifyInterceptor.java b/src/main/java/com/amplitude/api/IdentifyInterceptor.java
@@ -92,27 +92,6 @@ public void setIdentifyBatchIntervalMillis(long identifyBatchIntervalMillis) {
         this.identifyBatchIntervalMillis = identifyBatchIntervalMillis;
     }
 
-    private JSONObject fetchAndMergeToIdentifyEvent(JSONObject event) {
-        try {
-            List<JSONObject> identifys = dbHelper.getIdentifyInterceptors(lastIdentifyInterceptorId, -1);
-            if (identifys.isEmpty()) {
-                return event;
-            }
-            JSONObject identifyEventUserProperties = event.getJSONObject("user_properties");
-            JSONObject userProperties = mergeIdentifyInterceptList(identifys);
-            if (identifyEventUserProperties.has(Constants.AMP_OP_SET)) {
-                mergeUserProperties(userProperties, identifyEventUserProperties.getJSONObject(Constants.AMP_OP_SET));
-            }
-            identifyEventUserProperties.put(Constants.AMP_OP_SET, userProperties);
-            event.put("user_properties", identifyEventUserProperties);
-            dbHelper.removeIdentifyInterceptors(lastIdentifyInterceptorId);
-            return event;
-        } catch (JSONException e) {
-            AmplitudeLog.getLogger().w(TAG, "Identify Merge error: " + e.getMessage());
-        }
-        return event;
-    }
-
     private JSONObject getTransferIdentifyEvent() {
         try {
             List<JSONObject> identifys = dbHelper.getIdentifyInterceptors(lastIdentifyInterceptorId, -1);
@@ -154,22 +133,6 @@ public void transferInterceptedIdentify() {
         client.saveEvent(Constants.IDENTIFY_EVENT, identifyEvent);
     }
 
-    private JSONObject fetchAndMergeToNormalEvent(JSONObject event) {
-        try {
-            List<JSONObject> identifys = dbHelper.getIdentifyInterceptors(lastIdentifyInterceptorId, -1);
-            if (identifys.isEmpty()) {
-                return event;
-            }
-            JSONObject userProperties = mergeIdentifyInterceptList(identifys);
-            mergeUserProperties(userProperties, event.getJSONObject("user_properties"));
-            event.put("user_properties", userProperties);
-            dbHelper.removeIdentifyInterceptors(lastIdentifyInterceptorId);
-        } catch (JSONException e) {
-            AmplitudeLog.getLogger().w(TAG, "Identify Merge error: " + e.getMessage());
-        }
-        return event;
-    }
-
     private JSONObject mergeIdentifyInterceptList(List<JSONObject> identifys) throws JSONException {
         JSONObject userProperties = new JSONObject();
         for (JSONObject identify : identifys) {