[KYUUBI apache#6170] Merge JDBC Trino TPC jobs and Style Check jobs i…

…n GitHub Actions workflow # 🔍 Description We are asked by ASF Infra to limit the max-parallel up to 10 for each workflow. Merge JDBC Trino TPC jobs in GitHub Actions CI to reduce concurrency of workflow, also increase max-parallel of Spark jobs from 1 to 3 Also merge "shell check" job into "style check" job ## Types of changes 🔖 - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan 🧪 Pass GA. --- # Checklist 📝 - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes apache#6170 from pan3793/ci-2. Closes apache#6170 9bf5053 [Cheng Pan] merge shell check d0e7ece [Cheng Pan] nit b171695 [Cheng Pan] Tune CI Authored-by: Cheng Pan <[email protected]> Signed-off-by: Cheng Pan <[email protected]>
zhaohehuhu · Mar 21, 2024 · 291f201 · 291f201
1 parent af00ddc
commit 291f201
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 85 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -42,7 +42,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      max-parallel: 1
+      max-parallel: 3
       matrix:
         java:
           - 8
@@ -296,15 +296,13 @@ jobs:
             **/target/unit-tests.log
             **/kyuubi-hive-sql-engine.log*
 
-  jdbc-it:
-    name: JDBC Test
+  jdbc-trino-tpc-it:
+    name: JDBC Trino TPC Tests
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      max-parallel: 1
       matrix:
-        java:
-          - 8
+        java: [ 8 ]
         comment: [ "normal" ]
     steps:
       - uses: actions/checkout@v4
@@ -323,11 +321,17 @@ jobs:
         uses: ./.github/actions/setup-maven
       - name: Cache Engine Archives
         uses: ./.github/actions/cache-engine-archives
-      - name: Build and test JDBC with maven w/o linters
+      - name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters
         run: |
-          TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
+          JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it"
+          TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
+          TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch"
+          ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests
+          ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES} test
+          ./build/mvn ${MVN_OPT} -pl ${TRINO_TEST_MODULES} test
+          ./build/mvn ${MVN_OPT} -pl ${TPC_TEST_MODULES} test \
+          -Dmaven.plugin.scalatest.exclude.tags='' \
+          -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite
       - name: Upload test logs
         if: failure()
         uses: actions/upload-artifact@v3
@@ -336,78 +340,8 @@ jobs:
           path: |
             **/target/unit-tests.log
             **/kyuubi-jdbc-engine.log*
-
-  trino-it:
-    name: Trino Test
-    runs-on: ubuntu-22.04
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        java:
-          - 8
-        comment: [ "normal" ]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Free up disk space
-        run: ./.github/scripts/free_disk_space.sh
-      - name: Tune Runner VM
-        uses: ./.github/actions/tune-runner-vm
-      - name: Setup JDK ${{ matrix.java }}
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: ${{ matrix.java }}
-          cache: 'maven'
-          check-latest: false
-      - name: Setup Maven
-        uses: ./.github/actions/setup-maven
-      - name: Cache Engine Archives
-        uses: ./.github/actions/cache-engine-archives
-      - name: Build and test Trino with maven w/o linters
-        run: |
-          TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test
-      - name: Upload test logs
-        if: failure()
-        uses: actions/upload-artifact@v3
-        with:
-          name: unit-tests-log-java-${{ matrix.java }}-trino-${{ matrix.comment }}
-          path: |
-            **/target/unit-tests.log
             **/kyuubi-trino-engine.log*
 
-  tpch-tpcds:
-    name: TPC-H and TPC-DS Tests
-    runs-on: ubuntu-22.04
-    env:
-      SPARK_LOCAL_IP: localhost
-    steps:
-      - uses: actions/checkout@v4
-      - name: Free up disk space
-        run: ./.github/scripts/free_disk_space.sh
-      - name: Tune Runner VM
-        uses: ./.github/actions/tune-runner-vm
-      - name: Setup JDK 8
-        uses: actions/setup-java@v4
-        with:
-          distribution: temurin
-          java-version: 8
-          cache: 'maven'
-          check-latest: false
-      - name: Setup Maven
-        uses: ./.github/actions/setup-maven
-      - name: Cache Engine Archives
-        uses: ./.github/actions/cache-engine-archives
-      - name: Run TPC-DS Tests
-        run: |
-          TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch"
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests
-          ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test \
-          -Dmaven.plugin.scalatest.exclude.tags='' \
-          -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite
-
   kyuubi-on-k8s-it:
     name: Kyuubi Server On Kubernetes Integration Test
     runs-on: ubuntu-22.04

diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -107,11 +107,6 @@ jobs:
           echo "you need to manually fix it by following the information told by the command above"
           echo "---------------------------------------------------------------------------------"
 
-  shellcheck:
-    name: Super Linter and Shellcheck
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
       - name: Super Linter Checks
         uses: github/super-linter/slim@v5
         env: