Skip to content

Commit

Permalink
Remove Hive 2
Browse files Browse the repository at this point in the history
  • Loading branch information
manuzhang committed Dec 14, 2024
1 parent 2657377 commit 47ca107
Show file tree
Hide file tree
Showing 54 changed files with 110 additions and 10,467 deletions.
2 changes: 0 additions & 2 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ jmhJsonOutputPath=build/reports/jmh/results.json
jmhIncludeRegex=.*
systemProp.defaultFlinkVersions=1.20
systemProp.knownFlinkVersions=1.18,1.19,1.20
systemProp.defaultHiveVersions=2
systemProp.knownHiveVersions=2,4
systemProp.defaultSparkVersions=3.5
systemProp.knownSparkVersions=3.3,3.4,3.5
systemProp.defaultKafkaVersions=3
Expand Down
6 changes: 0 additions & 6 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ guava = "33.3.1-jre"
hadoop2 = "2.7.3"
hadoop3 = "3.4.1"
httpcomponents-httpclient5 = "5.4.1"
hive2 = { strictly = "2.3.9"} # see rich version usage explanation above
hive4 = "4.0.1"
immutables-value = "2.10.1"
jackson-bom = "2.18.2"
Expand Down Expand Up @@ -135,14 +134,9 @@ hadoop2-mapreduce-client-core = { module = "org.apache.hadoop:hadoop-mapreduce-c
hadoop2-minicluster = { module = "org.apache.hadoop:hadoop-minicluster", version.ref = "hadoop2" }
hadoop3-client = { module = "org.apache.hadoop:hadoop-client", version.ref = "hadoop3" }
hadoop3-common = { module = "org.apache.hadoop:hadoop-common", version.ref = "hadoop3" }
hive2-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive2" }
hive2-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive2" }
hive2-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive2" }
hive2-service = { module = "org.apache.hive:hive-service", version.ref = "hive2" }
hive4-exec = { module = "org.apache.hive:hive-exec", version.ref = "hive4" }
hive4-metastore = { module = "org.apache.hive:hive-metastore", version.ref = "hive4" }
hive4-standalone-metastore-server = { module = "org.apache.hive:hive-standalone-metastore-server", version.ref = "hive4" }
hive4-serde = { module = "org.apache.hive:hive-serde", version.ref = "hive4" }
hive4-service = { module = "org.apache.hive:hive-service", version.ref = "hive4" }
httpcomponents-httpclient5 = { module = "org.apache.httpcomponents.client5:httpclient5", version.ref = "httpcomponents-httpclient5" }
immutables-value = { module = "org.immutables:value", version.ref = "immutables-value" }
Expand Down
27 changes: 11 additions & 16 deletions mr/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -33,48 +33,43 @@ project(':iceberg-mr') {
implementation project(':iceberg-common')
implementation project(':iceberg-core')
api project(':iceberg-data')
implementation project(':iceberg-hive-metastore')
implementation project(':iceberg-orc')
implementation project(':iceberg-parquet')

compileOnly(libs.hadoop2.client) {
exclude group: 'org.apache.avro', module: 'avro'
}

compileOnly("${libs.hive2.exec.get().module}:${libs.hive2.exec.get().getVersion()}:core") {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
exclude group: 'com.google.guava'
implementation libs.caffeine
implementation libs.parquet.column
implementation("${libs.orc.core.get().module}:${libs.versions.orc.get()}:nohive") {
exclude group: 'org.apache.hadoop'
exclude group: 'commons-lang'
// These artifacts are shaded and included in the orc-core fat jar
exclude group: 'com.google.protobuf', module: 'protobuf-java'
exclude group: 'org.apache.avro', module: 'avro'
exclude group: 'org.apache.calcite.avatica'
exclude group: 'org.apache.hive', module: 'hive-llap-tez'
exclude group: 'org.apache.logging.log4j'
exclude group: 'org.pentaho' // missing dependency
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
exclude group: 'org.apache.hive', module: 'hive-storage-api'
}
compileOnly libs.hive2.metastore
compileOnly libs.hive2.serde

implementation libs.caffeine

testImplementation libs.calcite.core
testImplementation libs.calcite.druid

testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore')

testImplementation libs.avro.avro
testImplementation libs.calcite.core
testImplementation libs.kryo.shaded
testImplementation platform(libs.jackson.bom)
testImplementation libs.jackson.annotations
testImplementation(libs.hive2.service) {
testImplementation(libs.hive4.service) {
exclude group: 'org.apache.hive', module: 'hive-exec'
exclude group: 'org.pac4j', module: 'pac4j-saml-opensamlv3'
}
testImplementation libs.tez08.dag
testImplementation libs.tez08.mapreduce
testImplementation libs.junit.vintage.engine
}

test {
Expand Down

This file was deleted.

10 changes: 0 additions & 10 deletions mr/src/main/java/org/apache/iceberg/mr/InputFormatConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,6 @@ private InputFormatConfig() {}

public static final String CATALOG_CONFIG_PREFIX = "iceberg.catalog.";

public enum InMemoryDataModel {
HIVE,
GENERIC // Default data model is of Iceberg Generics
}

public static class ConfigBuilder {
private final Configuration conf;

Expand Down Expand Up @@ -163,11 +158,6 @@ public ConfigBuilder preferLocality() {
return this;
}

public ConfigBuilder useHiveRows() {
conf.set(IN_MEMORY_DATA_MODEL, InMemoryDataModel.HIVE.name());
return this;
}

/**
* Compute platforms pass down filters to data sources. If the data source cannot apply some
* filters, or only partially applies the filter, it will return the residual filter back. If
Expand Down
Loading

0 comments on commit 47ca107

Please sign in to comment.