From 4579fc2e936957ece459e6541c290a2bc2a9c6ac Mon Sep 17 00:00:00 2001 From: Abhishek Pandey <64667840+1abhishekpandey@users.noreply.github.com> Date: Fri, 6 Dec 2024 16:48:54 +0530 Subject: [PATCH] fix: handle invalid json issue (#486) * fix: handle invalid message json * refactor: use cleanUpEvents() * chore: add doc * fix: handle scenario with multiple invalid and valid events in a single batch Previously, Invalid events were not being marked as device mode processing done. --- .../android/sdk/core/FlushUtils.java | 32 ++++++++++++------- .../sdk/core/RudderCloudModeManager.java | 10 ++++-- .../sdk/core/RudderDeviceModeManager.java | 2 ++ 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/com/rudderstack/android/sdk/core/FlushUtils.java b/core/src/main/java/com/rudderstack/android/sdk/core/FlushUtils.java index 3fd23aa43..07c8c7418 100644 --- a/core/src/main/java/com/rudderstack/android/sdk/core/FlushUtils.java +++ b/core/src/main/java/com/rudderstack/android/sdk/core/FlushUtils.java @@ -13,6 +13,7 @@ import static com.rudderstack.android.sdk.core.RudderNetworkManager.Result; import static com.rudderstack.android.sdk.core.util.Utils.getBatch; import static com.rudderstack.android.sdk.core.util.Utils.getNumberOfBatches; +import androidx.annotation.Nullable; import android.text.TextUtils; @@ -88,6 +89,7 @@ static boolean flushToServer(int flushQueueSize, String dataPlaneUrl, DBPersiste } else { lastErrorMessage = ReportManager.LABEL_TYPE_PAYLOAD_NULL; + dbManager.markCloudModeDone(batchMessageIds); } RudderLogger.logWarn(String.format(Locale.US, "EventRepository: flush: Failed to send batch %d/%d retrying again, %d retries left", i, numberOfBatches, retries)); } @@ -132,6 +134,7 @@ private static void reportBatchesAndMessages(int numberOfBatches, int messagesSi * of deserialization and forming the payload object and creating the json string * again from the object * */ + @Nullable static String getPayloadFromMessages(List messageIds, List messages) { if (messageIds.isEmpty() || messages.isEmpty()) { RudderLogger.logWarn("FlushUtils: getPayloadFromMessages: Payload Construction failed: no messages to send"); @@ -159,22 +162,27 @@ static String getPayloadFromMessages(List messageIds, List mess String message = messages.get(index); // strip last ending object character message = message.substring(0, message.length() - 1); - // add sentAt time stamp - message = String.format("%s,\"sentAt\":\"%s\"},", message, sentAtTimestamp); - // add message size to batch size - messageSize = Utils.getUTF8Length(message); - totalBatchSize += messageSize; - // check batch size - if (totalBatchSize >= Utils.MAX_BATCH_SIZE) { - RudderLogger.logDebug(String.format(Locale.US, "FlushUtils: getPayloadFromMessages: MAX_BATCH_SIZE reached at index: %d | Total: %d", index, totalBatchSize)); - incrementDiscardedCounter(1, Collections.singletonMap(LABEL_TYPE, ReportManager.LABEL_TYPE_BATCH_SIZE_INVALID)); - break; + // Handle Invalid Message whose length is 0 + if (!message.isEmpty()) { + // add sentAt time stamp + message = String.format("%s,\"sentAt\":\"%s\"},", message, sentAtTimestamp); + // add message size to batch size + messageSize = Utils.getUTF8Length(message); + totalBatchSize += messageSize; + // check batch size + if (totalBatchSize >= Utils.MAX_BATCH_SIZE) { + RudderLogger.logDebug(String.format(Locale.US, "FlushUtils: getPayloadFromMessages: MAX_BATCH_SIZE reached at index: %d | Total: %d", index, totalBatchSize)); + incrementDiscardedCounter(1, Collections.singletonMap(LABEL_TYPE, ReportManager.LABEL_TYPE_BATCH_SIZE_INVALID)); + break; + } + // finally add message string to builder + batchMessagesBuilder.append(message); } - // finally add message string to builder - batchMessagesBuilder.append(message); // add message to batch ArrayLists batchMessageIds.add(messageIds.get(index)); } + // If the batchMessagesBuilder is empty, return null + if (batchMessagesBuilder.length() == 0) return null; if (batchMessagesBuilder.charAt(batchMessagesBuilder.length() - 1) == ',') { // remove trailing ',' batchMessagesBuilder.deleteCharAt(batchMessagesBuilder.length() - 1); diff --git a/core/src/main/java/com/rudderstack/android/sdk/core/RudderCloudModeManager.java b/core/src/main/java/com/rudderstack/android/sdk/core/RudderCloudModeManager.java index b0845970c..025037355 100644 --- a/core/src/main/java/com/rudderstack/android/sdk/core/RudderCloudModeManager.java +++ b/core/src/main/java/com/rudderstack/android/sdk/core/RudderCloudModeManager.java @@ -63,14 +63,15 @@ public void run() { RudderLogger.logInfo(String.format(Locale.US, "CloudModeManager: cloudModeProcessor: ServerResponse: %d", result.statusCode)); if (result.status == NetworkResponses.SUCCESS) { ReportManager.incrementCloudModeUploadSuccessCounter(messageIds.size()); - dbManager.markCloudModeDone(messageIds); - dbManager.runGcForEvents(); + cleanUpEvents(messageIds); exponentialBackOff.resetBackOff(); upTimeInMillis = Utils.getUpTimeInMillis(); sleepCount = Utils.getSleepDurationInSecond(upTimeInMillis, Utils.getUpTimeInMillis()); } else { incrementCloudModeUploadRetryCounter(1); } + } else { + cleanUpEvents(messageIds); } } } @@ -112,6 +113,11 @@ public void run() { }.start(); } + private void cleanUpEvents(List messageIds) { + dbManager.markCloudModeDone(messageIds); + dbManager.runGcForEvents(); + } + private void deleteEventsWithoutAnonymousId(ArrayList messages, ArrayList messageIds) { List eventsToDelete = new ArrayList<>(); for (int i = 0; i < messages.size(); i++) { diff --git a/core/src/main/java/com/rudderstack/android/sdk/core/RudderDeviceModeManager.java b/core/src/main/java/com/rudderstack/android/sdk/core/RudderDeviceModeManager.java index 668623323..dcf20c15c 100644 --- a/core/src/main/java/com/rudderstack/android/sdk/core/RudderDeviceModeManager.java +++ b/core/src/main/java/com/rudderstack/android/sdk/core/RudderDeviceModeManager.java @@ -246,6 +246,8 @@ private void replayMessageQueue() { RudderMessage message = RudderGson.deserialize(messages.get(i), RudderMessage.class); if (message != null) { processMessage(message, messageIds.get(i), true); + } else { + markDeviceModeTransformationDone(messageIds.get(i)); } } catch (Exception e) { ReportManager.reportError(e);