SNOW-1325562 Support Spark 3.5 (#557)

* Upgrade Spark * fix test * fix test * fix test * remove useless tests
snowflakedb · Jul 31, 2024 · 3b68d17 · 3b68d17
1 parent 8350590
commit 3b68d17
Show file tree

Hide file tree

Showing 29 changed files with 14 additions and 3,313 deletions.
diff --git a/.github/docker/build_image.sh b/.github/docker/build_image.sh
@@ -33,8 +33,8 @@ cd ../..
 
 # Build docker image
 docker build \
---build-arg SPARK_URL=https://archive.apache.org/dist/spark/spark-3.4.0/spark-3.4.0-bin-hadoop3.tgz \
---build-arg SPARK_BINARY_NAME=spark-3.4.0-bin-hadoop3.tgz \
+--build-arg SPARK_URL=https://archive.apache.org/dist/spark/spark-3.5.0/spark-3.5.0-bin-hadoop3.tgz \
+--build-arg SPARK_BINARY_NAME=spark-3.5.0-bin-hadoop3.tgz \
 --build-arg JDBC_URL=https://repo1.maven.org/maven2/net/snowflake/snowflake-jdbc/${TEST_JDBC_VERSION}/$JDBC_JAR_NAME \
 --build-arg JDBC_BINARY_NAME=$JDBC_JAR_NAME \
 --build-arg SPARK_CONNECTOR_LOCATION=target/scala-${TEST_SCALA_VERSION}/$SPARK_CONNECTOR_JAR_NAME \

diff --git a/.github/workflows/ClusterTest.yml b/.github/workflows/ClusterTest.yml
diff --git a/.github/workflows/IntegrationTest_2.12.yml b/.github/workflows/IntegrationTest_2.12.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
      matrix:
        scala_version: [ '2.12.11' ]
-       spark_version: [ '3.4.0' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'true', 'false' ]
        cloud_provider: [ 'aws', 'azure' ]
        # run_query_in_async can be removed after async mode is stable

diff --git a/.github/workflows/IntegrationTest_2.13.yml b/.github/workflows/IntegrationTest_2.13.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
      matrix:
        scala_version: [ '2.13.9' ]
-       spark_version: [ '3.4.0' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'true', 'false' ]
        cloud_provider: [ 'aws', 'azure' ]
        # run_query_in_async can be removed after async mode is stable

diff --git a/.github/workflows/IntegrationTest_gcp_2.12.yml b/.github/workflows/IntegrationTest_gcp_2.12.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
      matrix:
        scala_version: [ '2.12.11' ]
-       spark_version: [ '3.4.0' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'false' ]
        cloud_provider: [ 'gcp' ]
        # run_query_in_async can be removed after async mode is stable

diff --git a/.github/workflows/IntegrationTest_gcp_2.13.yml b/.github/workflows/IntegrationTest_gcp_2.13.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
      matrix:
        scala_version: [ '2.13.9' ]
-       spark_version: [ '3.4.0' ]
+       spark_version: [ '3.5.0' ]
        use_copy_unload: [ 'false' ]
        cloud_provider: [ 'gcp' ]
        # run_query_in_async can be removed after async mode is stable

diff --git a/ClusterTest/build.sbt b/ClusterTest/build.sbt
diff --git a/ClusterTest/build/sbt-launch-1.2.8.jar b/ClusterTest/build/sbt-launch-1.2.8.jar
diff --git a/ClusterTest/project/build.properties b/ClusterTest/project/build.properties
diff --git a/ClusterTest/project/plugins.sbt b/ClusterTest/project/plugins.sbt
diff --git a/ClusterTest/run_cluster_test.sh b/ClusterTest/run_cluster_test.sh
diff --git a/ClusterTest/src/main/python/ClusterTest.py b/ClusterTest/src/main/python/ClusterTest.py