From 2355ec2b16fb98af75b8132de0b445b91f71dc18 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 4 Nov 2024 23:12:16 -0800 Subject: [PATCH 1/2] Temp diff to enable NBCC for mdt --- .../apache/hudi/metadata/HoodieMetadataWriteUtils.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java index 1c9cdda2ba28..2dae970a6386 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java @@ -20,6 +20,7 @@ import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.client.FailOnFirstErrorWriteStatus; +import org.apache.hudi.client.transaction.lock.InProcessLockProvider; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieStorageConfig; import org.apache.hudi.common.config.RecordMergeMode; @@ -36,6 +37,7 @@ import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieCleanConfig; import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodiePayloadConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.metrics.HoodieMetricsConfig; @@ -92,7 +94,7 @@ public static HoodieWriteConfig createMetadataWriteConfig( .withAsyncClean(DEFAULT_METADATA_ASYNC_CLEAN) .withAutoClean(false) .withCleanerParallelism(MDT_DEFAULT_PARALLELISM) - .withFailedWritesCleaningPolicy(failedWritesCleaningPolicy) + .withFailedWritesCleaningPolicy(HoodieFailedWritesCleaningPolicy.LAZY) .withCleanerPolicy(dataTableCleaningPolicy); if (HoodieCleaningPolicy.KEEP_LATEST_COMMITS.equals(dataTableCleaningPolicy)) { @@ -165,7 +167,9 @@ public static HoodieWriteConfig createMetadataWriteConfig( .withRecordMergeStrategyId(HoodieRecordMerger.PAYLOAD_BASED_MERGE_STRATEGY_UUID) .withPayloadConfig(HoodiePayloadConfig.newBuilder() .withPayloadClass(HoodieMetadataPayload.class.getCanonicalName()).build()) - .withRecordMergeImplClasses(HoodieAvroRecordMerger.class.getCanonicalName()); + .withRecordMergeImplClasses(HoodieAvroRecordMerger.class.getCanonicalName()) + .withWriteConcurrencyMode(WriteConcurrencyMode.NON_BLOCKING_CONCURRENCY_CONTROL) + .withLockConfig(HoodieLockConfig.newBuilder().withLockProvider(InProcessLockProvider.class).build()); // RecordKey properties are needed for the metadata table records final Properties properties = new Properties(); From ed4bd8b4c0d796317a71f2e14b6849660d5fc780 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Tue, 5 Nov 2024 07:10:14 -0800 Subject: [PATCH 2/2] removing validations --- .../java/org/apache/hudi/config/HoodieWriteConfig.java | 7 +++++-- .../org/apache/hudi/metadata/HoodieMetadataWriteUtils.java | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 3aeaef5973c8..2aa4e5d8cce6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -3455,9 +3455,12 @@ private void validate() { writeConcurrencyMode.name())); } if (writeConcurrencyMode == WriteConcurrencyMode.NON_BLOCKING_CONCURRENCY_CONTROL) { + //checkArgument( + // writeConfig.getTableType().equals(HoodieTableType.MERGE_ON_READ) && writeConfig.isSimpleBucketIndex(), + // "Non-blocking concurrency control requires the MOR table with simple bucket index"); checkArgument( - writeConfig.getTableType().equals(HoodieTableType.MERGE_ON_READ) && writeConfig.isSimpleBucketIndex(), - "Non-blocking concurrency control requires the MOR table with simple bucket index"); + writeConfig.getTableType().equals(HoodieTableType.MERGE_ON_READ), + "Non-blocking concurrency control requires the MOR table"); } HoodieCleaningPolicy cleaningPolicy = HoodieCleaningPolicy.valueOf(writeConfig.getString(CLEANER_POLICY)); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java index 2dae970a6386..798860cd2492 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java @@ -29,6 +29,7 @@ import org.apache.hudi.common.model.HoodieCleaningPolicy; import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieRecordMerger; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.marker.MarkerType; @@ -175,6 +176,7 @@ public static HoodieWriteConfig createMetadataWriteConfig( final Properties properties = new Properties(); properties.put(HoodieTableConfig.RECORDKEY_FIELDS.key(), RECORD_KEY_FIELD_NAME); properties.put("hoodie.datasource.write.recordkey.field", RECORD_KEY_FIELD_NAME); + properties.put(HoodieTableConfig.TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); if (nonEmpty(writeConfig.getMetricReporterMetricsNamePrefix())) { properties.put(HoodieMetricsConfig.METRICS_REPORTER_PREFIX.key(), writeConfig.getMetricReporterMetricsNamePrefix() + METADATA_TABLE_NAME_SUFFIX);