diff --git a/pom.xml b/pom.xml index 806f3768..eef7afd1 100644 --- a/pom.xml +++ b/pom.xml @@ -159,6 +159,11 @@ parquet-column ${parquet.version} + + org.apache.parquet + parquet-hadoop + ${parquet.version} + diff --git a/xtable-hudi/pom.xml b/xtable-hudi/pom.xml index ebc5d3e6..c5d80737 100644 --- a/xtable-hudi/pom.xml +++ b/xtable-hudi/pom.xml @@ -63,6 +63,14 @@ org.apache.parquet parquet-column + + org.apache.parquet + parquet-avro + + + org.apache.parquet + parquet-hadoop + @@ -73,12 +81,6 @@ org.apache.hudi hudi-java-client - - - org.apache.hbase - hbase-server - - @@ -181,6 +183,7 @@ shade + true @@ -205,14 +208,30 @@ com.fasterxml.jackson.core:jackson-databind com.fasterxml.jackson.datatype:jackson-datatype-jsr310 com.fasterxml.jackson.core:jackson-core + com.fasterxml.jackson.core:jackson-annotations org.apache.parquet:parquet-column + org.apache.parquet:parquet-avro org.apache.parquet:parquet-common org.apache.parquet:parquet-encoding + org.apache.parquet:parquet-hadoop + org.apache.parquet:parquet-format-structures org.apache.hudi:hudi-java-client org.apache.hudi:hudi-client-common org.apache.hudi:hudi-common org.apache.avro:avro com.google.guava:guava + + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-metrics + org.apache.hbase:hbase-metrics-api + org.apache.hbase:hbase-protocol + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-server + org.apache.hbase.thirdparty:hbase-shaded-miscellaneous + org.apache.hbase.thirdparty:hbase-shaded-netty + org.apache.hbase.thirdparty:hbase-shaded-protobuf @@ -223,6 +242,17 @@ org.apache.hudi. org.apache.xtable.shade.org.apache.hudi. + + org.apache.hudi.io.storage.HoodieHBaseKVComparator + + + + org.apache.hadoop.hbase. + org.apache.xtable.shade.org.apache.hadoop.hbase. + + org.apache.hadoop.hbase.CellComparator + org.apache.hadoop.hbase.CellComparatorImpl + org.apache.avro. diff --git a/xtable-hudi/src/main/java/org/apache/xtable/hbase/NoOpMetricsRegionServerSourceFactory.java b/xtable-hudi/src/main/java/org/apache/xtable/hbase/NoOpMetricsRegionServerSourceFactory.java new file mode 100644 index 00000000..b8419ccb --- /dev/null +++ b/xtable-hudi/src/main/java/org/apache/xtable/hbase/NoOpMetricsRegionServerSourceFactory.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.xtable.hbase; + +import org.apache.hadoop.hbase.io.MetricsIOSource; +import org.apache.hadoop.hbase.io.MetricsIOWrapper; +import org.apache.hadoop.hbase.regionserver.MetricsHeapMemoryManagerSource; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSource; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory; +import org.apache.hadoop.hbase.regionserver.MetricsRegionServerWrapper; +import org.apache.hadoop.hbase.regionserver.MetricsRegionSource; +import org.apache.hadoop.hbase.regionserver.MetricsRegionWrapper; +import org.apache.hadoop.hbase.regionserver.MetricsTableAggregateSource; +import org.apache.hadoop.hbase.regionserver.MetricsTableSource; +import org.apache.hadoop.hbase.regionserver.MetricsTableWrapperAggregate; +import org.apache.hadoop.hbase.regionserver.MetricsUserAggregateSource; +import org.apache.hadoop.hbase.regionserver.MetricsUserSource; + +/** Provides a No-Op metrics implementation for the HFile required by Hudi. */ +public class NoOpMetricsRegionServerSourceFactory implements MetricsRegionServerSourceFactory { + @Override + public MetricsRegionServerSource createServer(MetricsRegionServerWrapper regionServerWrapper) { + return null; + } + + @Override + public MetricsRegionSource createRegion(MetricsRegionWrapper wrapper) { + return null; + } + + @Override + public MetricsUserSource createUser(String shortUserName) { + return null; + } + + @Override + public MetricsUserAggregateSource getUserAggregate() { + return null; + } + + @Override + public MetricsTableSource createTable(String table, MetricsTableWrapperAggregate wrapper) { + return null; + } + + @Override + public MetricsTableAggregateSource getTableAggregate() { + return null; + } + + @Override + public MetricsHeapMemoryManagerSource getHeapMemoryManager() { + return null; + } + + @Override + public MetricsIOSource createIO(MetricsIOWrapper wrapper) { + return new NoOpMetricsIOSource(); + } + + private static class NoOpMetricsIOSource implements MetricsIOSource { + + @Override + public void updateFsReadTime(long t) {} + + @Override + public void updateFsPReadTime(long t) {} + + @Override + public void updateFsWriteTime(long t) {} + + @Override + public void init() {} + + @Override + public void setGauge(String gaugeName, long value) {} + + @Override + public void incGauge(String gaugeName, long delta) {} + + @Override + public void decGauge(String gaugeName, long delta) {} + + @Override + public void removeMetric(String key) {} + + @Override + public void incCounters(String counterName, long delta) {} + + @Override + public void updateHistogram(String name, long value) {} + + @Override + public String getMetricsContext() { + return ""; + } + + @Override + public String getMetricsDescription() { + return ""; + } + + @Override + public String getMetricsJmxContext() { + return ""; + } + + @Override + public String getMetricsName() { + return ""; + } + } +} diff --git a/xtable-hudi/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory b/xtable-hudi/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory new file mode 100644 index 00000000..dab50e04 --- /dev/null +++ b/xtable-hudi/src/main/resources/META-INF/services/org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.xtable.hbase.NoOpMetricsRegionServerSourceFactory diff --git a/xtable-hudi/src/main/resources/META-INF/services/org.apache.xtable.shade.org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory b/xtable-hudi/src/main/resources/META-INF/services/org.apache.xtable.shade.org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory new file mode 100644 index 00000000..dab50e04 --- /dev/null +++ b/xtable-hudi/src/main/resources/META-INF/services/org.apache.xtable.shade.org.apache.hadoop.hbase.regionserver.MetricsRegionServerSourceFactory @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.xtable.hbase.NoOpMetricsRegionServerSourceFactory diff --git a/xtable-integration-tests/pom.xml b/xtable-integration-tests/pom.xml index cf960b2e..eb11cf1b 100644 --- a/xtable-integration-tests/pom.xml +++ b/xtable-integration-tests/pom.xml @@ -52,6 +52,18 @@ hudi-spark${spark.version.prefix}-bundle_${scala.binary.version} test + + org.apache.hudi + hudi-java-client + + + org.apache.hbase + hbase-server + + + test + + org.apache.iceberg @@ -119,18 +131,21 @@ org.apache.xtable xtable-delta_${scala.binary.version} ${project.version} + bundled test org.apache.xtable xtable-iceberg ${project.version} + bundled test org.apache.xtable xtable-hudi ${project.version} + bundled test diff --git a/xtable-integration-tests/src/test/java/org/apache/xtable/ITConversionController.java b/xtable-integration-tests/src/test/java/org/apache/xtable/ITConversionController.java index 479eea7e..3922b22e 100644 --- a/xtable-integration-tests/src/test/java/org/apache/xtable/ITConversionController.java +++ b/xtable-integration-tests/src/test/java/org/apache/xtable/ITConversionController.java @@ -70,9 +70,7 @@ import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.table.timeline.HoodieInstant; -import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; import org.apache.iceberg.hadoop.HadoopTables; @@ -98,6 +96,8 @@ import org.apache.xtable.iceberg.IcebergConversionSourceProvider; import org.apache.xtable.model.storage.TableFormat; import org.apache.xtable.model.sync.SyncMode; +import org.apache.xtable.shade.org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.xtable.shade.org.apache.iceberg.Snapshot; public class ITConversionController { @TempDir public static Path tempDir; diff --git a/xtable-integration-tests/src/test/java/org/apache/xtable/loadtest/LoadTest.java b/xtable-integration-tests/src/test/java/org/apache/xtable/loadtest/LoadTest.java index 63c83bda..b376d2c5 100644 --- a/xtable-integration-tests/src/test/java/org/apache/xtable/loadtest/LoadTest.java +++ b/xtable-integration-tests/src/test/java/org/apache/xtable/loadtest/LoadTest.java @@ -34,7 +34,6 @@ import org.junit.jupiter.api.io.TempDir; import org.apache.hudi.common.model.HoodieTableType; -import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.xtable.GenericTable; @@ -47,6 +46,7 @@ import org.apache.xtable.hudi.TestJavaHudiTable; import org.apache.xtable.model.storage.TableFormat; import org.apache.xtable.model.sync.SyncMode; +import org.apache.xtable.shade.org.apache.hudi.common.table.timeline.HoodieInstant; /** * Tests that can be run manually to simulate lots of commits/partitions/files/etc. to understand