diff --git a/build.gradle b/build.gradle index eb63ce138ab2..54fdd61f699c 100644 --- a/build.gradle +++ b/build.gradle @@ -119,6 +119,8 @@ allprojects { repositories { mavenCentral() mavenLocal() + // Spark 3.5.4 RC2 staging repo + maven { url = "https://repository.apache.org/content/repositories/orgapachespark-1473/" } } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 16bafc7a736a..1a2eea96ab11 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ slf4j = "2.0.16" snowflake-jdbc = "3.20.0" spark-hive33 = "3.3.4" spark-hive34 = "3.4.4" -spark-hive35 = "3.5.2" +spark-hive35 = "3.5.4" sqlite-jdbc = "3.47.1.0" testcontainers = "1.20.4" tez010 = "0.10.4" diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java index 38ec3a0e838c..9b22a308d6b5 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java @@ -59,6 +59,11 @@ public void close() { accessor.close(); } + // If a column vector is writable or constant, it should override this method and do nothing. + // See more details at SPARK-50235, SPARK-50463 (Fixed in Spark 3.5.4) + @Override + public void closeIfFreeable() {} + @Override public boolean hasNull() { return nullabilityHolder.hasNulls();