remove spark-authz-paimon-test profile, use 3.4 as spark binary versi…

…on for Paimon
apache · Oct 18, 2023 · 8393f6e · 8393f6e
1 parent a4e01a2
commit 8393f6e
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 35 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -77,22 +77,7 @@ jobs:
             spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6'
             exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest'
             comment: 'verify-on-spark-3.5-binary'
-          - java: 8
-            spark: '3.5'
-            deactivate-profiles: '-P !spark-authz-paimon-test'
-          - java: 11
-            spark: '3.5'
-            deactivate-profiles: '-P !spark-authz-paimon-test'
-          - java: 17
-            spark: '3.5'
-            deactivate-profiles: '-P !spark-authz-paimon-test'
         exclude:
-          - java: 8
-            spark: '3.5'
-          - java: 11
-            spark: '3.5'
-          - java: 17
-            spark: '3.5'
           # SPARK-33772: Spark supports JDK 17 since 3.3.0
           - java: 17
             spark: '3.1'
@@ -123,10 +108,7 @@ jobs:
         run: |
           TEST_MODULES="dev/kyuubi-codecov"
           ./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \
-          -Pspark-${{ matrix.spark }} \
-          -Pspark-authz-hudi-test -Pspark-authz-paimon-test \
-          ${{ matrix.deactivate-profiles }} \
-          ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }}
+          -Pspark-${{ matrix.spark }} -Pspark-authz-hudi-test ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }}
       - name: Code coverage
         if: |
           matrix.java == 8 &&

diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml
@@ -323,6 +323,12 @@
             <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-spark-${paimon.spark.binary.version}</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -353,21 +359,6 @@
             </dependencies>
         </profile>
 
-        <!--
-          Add spark-authz-paimon-test profile here to avoid import Apache Paimon when enable spark-3.5.
-          Can remove this profile when Apache Paimon supports Spark 3.5
-          -->
-        <profile>
-            <id>spark-authz-paimon-test</id>
-            <dependencies>
-                <dependency>
-                    <groupId>org.apache.paimon</groupId>
-                    <artifactId>paimon-spark-${spark.binary.version}</artifactId>
-                    <scope>test</scope>
-                </dependency>
-            </dependencies>
-        </profile>
-
         <profile>
             <id>gen-policy</id>
             <build>

diff --git a/pom.xml b/pom.xml
@@ -185,6 +185,7 @@
         <netty.version>4.1.93.Final</netty.version>
         <openai.java.version>0.12.0</openai.java.version>
         <paimon.version>0.5.0-incubating</paimon.version>
+        <paimon.spark.binary.version>${spark.binary.version}</paimon.spark.binary.version>
         <parquet.version>1.10.1</parquet.version>
         <phoenix.version>6.0.0</phoenix.version>
         <prometheus.version>0.16.0</prometheus.version>
@@ -1488,7 +1489,7 @@
 
             <dependency>
                 <groupId>org.apache.paimon</groupId>
-                <artifactId>paimon-spark-${spark.binary.version}</artifactId>
+                <artifactId>paimon-spark-${paimon.spark.binary.version}</artifactId>
                 <version>${paimon.version}</version>
             </dependency>
         </dependencies>
@@ -2273,6 +2274,8 @@
                 <delta.version>3.0.0</delta.version>
                 <!-- Remove this when Hudi supports Spark 3.5 -->
                 <hudi.spark.binary.version>3.4</hudi.spark.binary.version>
+                <!-- Remove this when Paimon supports Spark 3.5 -->
+                <paimon.spark.binary.version>3.4</paimon.spark.binary.version>
                 <spark.version>3.5.0</spark.version>
                 <spark.binary.version>3.5</spark.binary.version>
                 <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.PySparkTest,org.apache.kyuubi.tags.PaimonTest</maven.plugin.scalatest.exclude.tags>