From a0361bce124951cb4451a294e252ddeb2ce030e3 Mon Sep 17 00:00:00 2001 From: Timothy Brown Date: Sun, 7 Jul 2024 20:22:25 -0500 Subject: [PATCH] fix compile issues, update config naming --- .../xtable/conversion/TableSyncConfig.java | 22 +++++++++-- .../apache/xtable/hudi/HudiSourceConfig.java | 4 +- .../apache/xtable/ITConversionController.java | 4 +- .../xtable/hudi/sync/XTableSyncTool.java | 39 ++++++++++++------- 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/TableSyncConfig.java b/xtable-api/src/main/java/org/apache/xtable/conversion/TableSyncConfig.java index 2166899db..24874906e 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/TableSyncConfig.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/TableSyncConfig.java @@ -26,14 +26,30 @@ import lombok.NonNull; import lombok.Value; +import com.google.common.base.Preconditions; + import org.apache.xtable.model.sync.SyncMode; @Value -@Builder(toBuilder = true) public class TableSyncConfig { // TODO add docs for all fields @NonNull SourceTable sourceTable; List targetTables; - @Builder.Default SyncMode syncMode = SyncMode.INCREMENTAL; - @Builder.Default Map properties = Collections.emptyMap(); + SyncMode syncMode; + Map properties; + + @Builder + TableSyncConfig( + @NonNull SourceTable sourceTable, + List targetTables, + SyncMode syncMode, + Map properties) { + this.sourceTable = sourceTable; + this.targetTables = targetTables; + Preconditions.checkArgument( + targetTables != null && !targetTables.isEmpty(), + "Please provide at-least one format to sync"); + this.syncMode = syncMode == null ? SyncMode.INCREMENTAL : syncMode; + this.properties = properties == null ? Collections.emptyMap() : properties; + } } diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java index 02328f397..944cd2212 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/HudiSourceConfig.java @@ -35,9 +35,9 @@ @Value public class HudiSourceConfig { public static final String PARTITION_SPEC_EXTRACTOR_CLASS = - "onetable.hudi.source.partition_spec_extractor_class"; + "xtable.hudi.source.partition_spec_extractor_class"; public static final String PARTITION_FIELD_SPEC_CONFIG = - "onetable.hudi.source.partition_field_spec_config"; + "xtable.hudi.source.partition_field_spec_config"; String partitionSpecExtractorClass; List partitionFieldSpecs; diff --git a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java index 36b060b27..85a78979c 100644 --- a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java @@ -850,7 +850,7 @@ private static TableSyncConfig getTableSyncConfig( String tableName, GenericTable table, List targetTableFormats, - String oneTablePartitionConfig, + String partitionConfig, Integer metadataRetentionInHours) { SourceTable sourceTable = SourceTable.builder() @@ -876,7 +876,7 @@ private static TableSyncConfig getTableSyncConfig( .sourceTable(sourceTable) .targetTables(targetTables) .syncMode(syncMode) - .properties(Collections.singletonMap(PARTITION_FIELD_SPEC_CONFIG, oneTablePartitionConfig)) + .properties(Collections.singletonMap(PARTITION_FIELD_SPEC_CONFIG, partitionConfig)) .build(); } } diff --git a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java index 3b9837bac..d565ced03 100644 --- a/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java +++ b/xtable-hudi-support/xtable-hudi-support-extensions/src/main/java/org/apache/xtable/hudi/sync/XTableSyncTool.java @@ -18,6 +18,9 @@ package org.apache.xtable.hudi.sync; +import static org.apache.xtable.hudi.HudiSourceConfig.PARTITION_FIELD_SPEC_CONFIG; + +import java.time.Duration; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -35,9 +38,10 @@ import org.apache.hudi.sync.common.HoodieSyncTool; import org.apache.xtable.conversion.ConversionController; +import org.apache.xtable.conversion.SourceTable; import org.apache.xtable.conversion.TableSyncConfig; +import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.hudi.HudiConversionSourceProvider; -import org.apache.xtable.hudi.HudiSourceConfigImpl; import org.apache.xtable.model.schema.PartitionTransformType; import org.apache.xtable.model.sync.SyncMode; import org.apache.xtable.model.sync.SyncResult; @@ -54,7 +58,7 @@ public XTableSyncTool(Properties props, Configuration hadoopConf) { super(props, hadoopConf); this.config = new XTableSyncConfig(props); this.hudiConversionSourceProvider = new HudiConversionSourceProvider(); - hudiConversionSourceProvider.init(hadoopConf, Collections.emptyMap()); + hudiConversionSourceProvider.init(hadoopConf); } @Override @@ -65,18 +69,27 @@ public void syncHoodieTable() { .collect(Collectors.toList()); String basePath = config.getString(HoodieSyncConfig.META_SYNC_BASE_PATH); String tableName = config.getString(HoodieTableConfig.HOODIE_TABLE_NAME_KEY); + SourceTable sourceTable = SourceTable.builder().name(tableName).metadataPath(basePath).build(); + List targetTables = + formatsToSync.stream() + .map( + format -> + TargetTable.builder() + .metadataPath(basePath) + .metadataRetention( + Duration.ofHours( + config.getInt( + XTableSyncConfig.ONE_TABLE_TARGET_METADATA_RETENTION_HOURS))) + .formatName(format) + .build()) + .collect(Collectors.toList()); TableSyncConfig tableSyncConfig = - TableSyncConfigImpl.builder() - .tableName(tableName) - .tableBasePath(basePath) - .targetTableFormats(formatsToSync) - .hudiSourceConfig( - HudiSourceConfigImpl.builder() - .partitionFieldSpecConfig(getPartitionSpecConfig()) - .build()) + TableSyncConfig.builder() + .sourceTable(sourceTable) + .targetTables(targetTables) .syncMode(SyncMode.INCREMENTAL) - .targetMetadataRetentionInHours( - config.getInt(XTableSyncConfig.ONE_TABLE_TARGET_METADATA_RETENTION_HOURS)) + .properties( + Collections.singletonMap(PARTITION_FIELD_SPEC_CONFIG, getPartitionSpecConfig())) .build(); Map results = new ConversionController(hadoopConf).sync(tableSyncConfig, hudiConversionSourceProvider); @@ -86,7 +99,7 @@ public void syncHoodieTable() { entry -> entry.getValue().getStatus().getStatusCode() != SyncResult.SyncStatusCode.SUCCESS) - .map(entry -> entry.getKey().toString()) + .map(Map.Entry::getKey) .collect(Collectors.joining(",")); if (!failingFormats.isEmpty()) { throw new HoodieException("Unable to sync to InternalTable for formats: " + failingFormats);