diff --git a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java index 08b0eca90907..7a276d625412 100644 --- a/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java +++ b/pinot-core/src/main/java/org/apache/pinot/core/common/MinionConstants.java @@ -219,6 +219,16 @@ public static class UpsertCompactionTask { */ public static final String SNAPSHOT = "snapshot"; + /** + * key representing if upsert compaction task executor should ignore crc mismatch or not during task execution + */ + public static final String IGNORE_CRC_MISMATCH_KEY = "ignoreCrcMismatch"; + + /** + * default value for the key IGNORE_CRC_MISMATCH_KEY: false + */ + public static final boolean DEFAULT_IGNORE_CRC_MISMATCH = false; + /** * number of segments to query in one batch to fetch valid doc id metadata, by default 500 */ diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java index 12f9ee12bbec..e5469a22ae6e 100644 --- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java +++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskExecutor.java @@ -26,6 +26,7 @@ import org.apache.pinot.common.metrics.MinionMeter; import org.apache.pinot.common.restlet.resources.ValidDocIdsType; import org.apache.pinot.core.common.MinionConstants; +import org.apache.pinot.core.common.MinionConstants.UpsertCompactionTask; import org.apache.pinot.core.minion.PinotTaskConfig; import org.apache.pinot.plugin.minion.tasks.BaseSingleSegmentConversionExecutor; import org.apache.pinot.plugin.minion.tasks.MinionTaskUtils; @@ -58,11 +59,13 @@ protected SegmentConversionResult convert(PinotTaskConfig pinotTaskConfig, File TableConfig tableConfig = getTableConfig(tableNameWithType); String validDocIdsTypeStr = - configs.getOrDefault(MinionConstants.UpsertCompactionTask.VALID_DOC_IDS_TYPE, ValidDocIdsType.SNAPSHOT.name()); + configs.getOrDefault(UpsertCompactionTask.VALID_DOC_IDS_TYPE, ValidDocIdsType.SNAPSHOT.name()); SegmentMetadataImpl segmentMetadata = new SegmentMetadataImpl(indexDir); String originalSegmentCrcFromTaskGenerator = configs.get(MinionConstants.ORIGINAL_SEGMENT_CRC_KEY); String crcFromDeepStorageSegment = segmentMetadata.getCrc(); - if (!originalSegmentCrcFromTaskGenerator.equals(crcFromDeepStorageSegment)) { + boolean ignoreCrcMismatch = Boolean.parseBoolean(configs.getOrDefault(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY, + String.valueOf(UpsertCompactionTask.DEFAULT_IGNORE_CRC_MISMATCH))); + if (!ignoreCrcMismatch && !originalSegmentCrcFromTaskGenerator.equals(crcFromDeepStorageSegment)) { String message = String.format("Crc mismatched between ZK and deepstore copy of segment: %s. Expected crc " + "from ZK: %s, crc from deepstore: %s", segmentName, originalSegmentCrcFromTaskGenerator, crcFromDeepStorageSegment); @@ -145,7 +148,7 @@ private static SegmentGeneratorConfig getSegmentGeneratorConfig(File workingDir, protected SegmentZKMetadataCustomMapModifier getSegmentZKMetadataCustomMapModifier(PinotTaskConfig pinotTaskConfig, SegmentConversionResult segmentConversionResult) { return new SegmentZKMetadataCustomMapModifier(SegmentZKMetadataCustomMapModifier.ModifyMode.UPDATE, - Collections.singletonMap(MinionConstants.UpsertCompactionTask.TASK_TYPE + MinionConstants.TASK_TIME_SUFFIX, + Collections.singletonMap(UpsertCompactionTask.TASK_TYPE + MinionConstants.TASK_TIME_SUFFIX, String.valueOf(System.currentTimeMillis()))); } } diff --git a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java index 2fa814db0131..77aaefc069d3 100644 --- a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java +++ b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/main/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGenerator.java @@ -185,6 +185,9 @@ public List generateTasks(List tableConfigs) { configs.put(MinionConstants.UPLOAD_URL_KEY, _clusterInfoAccessor.getVipUrl() + "/segments"); configs.put(MinionConstants.ORIGINAL_SEGMENT_CRC_KEY, String.valueOf(segment.getCrc())); configs.put(UpsertCompactionTask.VALID_DOC_IDS_TYPE, validDocIdsType.toString()); + configs.put(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY, + taskConfigs.getOrDefault(UpsertCompactionTask.IGNORE_CRC_MISMATCH_KEY, + String.valueOf(UpsertCompactionTask.DEFAULT_IGNORE_CRC_MISMATCH))); pinotTaskConfigs.add(new PinotTaskConfig(UpsertCompactionTask.TASK_TYPE, configs)); numTasks++; }