Skip to content

Commit

Permalink
remove unused dependencies, start building bundles
Browse files Browse the repository at this point in the history
  • Loading branch information
@the-other-tim-brown
the-other-tim-brown committed Jan 8, 2025
1 parent b0c1b50 commit 6d7b0dc
Show file tree
Hide file tree
Showing 7 changed files with 289 additions and 168 deletions.
11 changes: 11 additions & 0 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,11 @@
<artifactId>parquet-avro</artifactId>
<version>${parquet.version}</version>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
<version>${parquet.version}</version>
</dependency>

<!-- Logging -->
<dependency>
Expand Down Expand Up @@ -322,6 +327,12 @@
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>commons-cli</groupId>
Expand Down
37 changes: 6 additions & 31 deletions xtable-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,33 +34,6 @@
<artifactId>xtable-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.xtable</groupId>
<artifactId>xtable-hudi-support-utils</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<!-- Scala dependencies -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
</dependency>

<!-- Hadoop dependencies -->
<dependency>
Expand All @@ -74,10 +47,6 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>

<!-- Mockito -->
<dependency>
Expand Down Expand Up @@ -114,6 +83,12 @@
<artifactId>log4j-slf4j2-impl</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
86 changes: 77 additions & 9 deletions xtable-delta/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,18 +50,18 @@
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>

<!-- Avro -->
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>

<!-- Scala dependencies -->
Expand Down Expand Up @@ -93,17 +93,19 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>

<!-- Spark/Iceberg/Hudi dependencies for reading/writing tables -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
Expand Down Expand Up @@ -169,6 +171,72 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"> </transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
<resource>LICENSE</resource>
<resource>NOTICE</resource>
<resource>NOTICE.txt</resource>
</resources>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
</transformers>
<artifactSet>
<includes combine.children="append">
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>com.fasterxml.jackson.core:jackson-databind</include>
<include>com.fasterxml.jackson.core:jackson-annotations</include>
<include>org.scala-lang:scala-library</include>
<include>io.delta:delta-core_${scala.binary.version}</include>
<include>io.delta:delta-storage</include>
<include>org.apache.commons:commons-lang3</include>
<include>com.google.guava:guava</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>com.fasterxml.jackson.</pattern>
<shadedPattern>org.apache.xtable.shade.com.fasterxml.jackson.</shadedPattern>
</relocation>
<relocation>
<pattern>io.delta.</pattern>
<shadedPattern>org.apache.xtable.shade.io.delta.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.spark.sql.delta.</pattern>
<shadedPattern>org.apache.xtable.shade.org.apache.spark.sql.delta.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.commons.</pattern>
<shadedPattern>org.apache.xtable.shade.org.apache.commons.</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.guava.</pattern>
<shadedPattern>org.apache.xtable.shade.com.google.guava.</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
111 changes: 90 additions & 21 deletions xtable-hudi/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,18 +46,7 @@
<artifactId>xtable-hudi-support-utils</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand All @@ -69,19 +58,27 @@
<artifactId>avro</artifactId>
</dependency>

<!-- Parquet -->
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-column</artifactId>
</dependency>

<!-- Hudi dependencies -->
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-spark${spark.version.prefix}-bundle_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-java-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- Hadoop dependencies -->
Expand All @@ -96,12 +93,8 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>

<!-- Spark/Iceberg/Hudi dependencies for reading/writing tables -->
<!-- Hudi dependencies for reading/writing tables -->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
Expand Down Expand Up @@ -149,6 +142,12 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.xtable</groupId>
<artifactId>xtable-core_${scala.binary.version}</artifactId>
Expand All @@ -172,6 +171,76 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer"> </transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resources>
<resource>LICENSE</resource>
<resource>NOTICE</resource>
<resource>NOTICE.txt</resource>
</resources>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>target/classes/META-INF/LICENSE</file>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>target/classes/META-INF/NOTICE</file>
</transformer>
</transformers>
<artifactSet>
<includes combine.children="append">
<include>org.apache.xtable:xtable-hudi-support-utils</include>
<include>com.fasterxml.jackson.core:jackson-databind</include>
<include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310</include>
<include>com.fasterxml.jackson.core:jackson-core</include>
<include>org.apache.parquet:parquet-column</include>
<include>org.apache.parquet:parquet-common</include>
<include>org.apache.parquet:parquet-encoding</include>
<include>org.apache.hudi:hudi-java-client</include>
<include>org.apache.hudi:hudi-client-common</include>
<include>org.apache.hudi:hudi-common</include>
<include>org.apache.avro:avro</include>
<include>com.google.guava:guava</include>
</includes>
</artifactSet>
<relocations combine.children="append">
<relocation>
<pattern>org.apache.parquet.</pattern>
<shadedPattern>org.apache.xtable.shade.org.apache.parquet.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hudi.</pattern>
<shadedPattern>org.apache.xtable.shade.org.apache.hudi.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.avro.</pattern>
<shadedPattern>org.apache.xtable.shade.org.apache.avro.</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.guava.</pattern>
<shadedPattern>org.apache.xtable.shade.com.google.guava.</shadedPattern>
</relocation>
<relocation>
<pattern>com.fasterxml.jackson.</pattern>
<shadedPattern>org.apache.xtable.shade.com.fasterxml.jackson.</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

Expand Down
2 changes: 0 additions & 2 deletions xtable-hudi/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
import org.apache.parquet.avro.AvroParquetWriter;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.util.HadoopOutputFile;
import org.jetbrains.annotations.NotNull;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

Expand Down Expand Up @@ -447,7 +446,6 @@ private GenericRecord createRecord(
return record;
}

@NotNull
private GenericData.Record getNestedRecord(Integer nestedIntValue) {
GenericData.Record nested = new GenericData.Record(NESTED_SCHEMA);
nested.put("nested_int", nestedIntValue);
Expand Down
Loading

0 comments on commit 6d7b0dc

Please sign in to comment.