From 291f201a9b5d7aef1daf5153145c71f12137b141 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Tue, 12 Mar 2024 15:22:22 +0800 Subject: [PATCH] [KYUUBI #6170] Merge JDBC Trino TPC jobs and Style Check jobs in GitHub Actions workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # :mag: Description We are asked by ASF Infra to limit the max-parallel up to 10 for each workflow. Merge JDBC Trino TPC jobs in GitHub Actions CI to reduce concurrency of workflow, also increase max-parallel of Spark jobs from 1 to 3 Also merge "shell check" job into "style check" job ## Types of changes :bookmark: - [ ] Bugfix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) ## Test Plan ๐Ÿงช Pass GA. --- # Checklist ๐Ÿ“ - [x] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html) **Be nice. Be informative.** Closes #6170 from pan3793/ci-2. Closes #6170 9bf505321 [Cheng Pan] merge shell check d0e7ece54 [Cheng Pan] nit b17169573 [Cheng Pan] Tune CI Authored-by: Cheng Pan Signed-off-by: Cheng Pan --- .github/workflows/master.yml | 94 ++++++------------------------------ .github/workflows/style.yml | 5 -- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 8b7a790e6f7..bc82025edf3 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-22.04 strategy: fail-fast: false - max-parallel: 1 + max-parallel: 3 matrix: java: - 8 @@ -296,15 +296,13 @@ jobs: **/target/unit-tests.log **/kyuubi-hive-sql-engine.log* - jdbc-it: - name: JDBC Test + jdbc-trino-tpc-it: + name: JDBC Trino TPC Tests runs-on: ubuntu-22.04 strategy: fail-fast: false - max-parallel: 1 matrix: - java: - - 8 + java: [ 8 ] comment: [ "normal" ] steps: - uses: actions/checkout@v4 @@ -323,11 +321,17 @@ jobs: uses: ./.github/actions/setup-maven - name: Cache Engine Archives uses: ./.github/actions/cache-engine-archives - - name: Build and test JDBC with maven w/o linters + - name: Build and test JDBC and Trino engines, TPC-H/TPC-DS with maven w/o linters run: | - TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test + JDBC_TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" + TRINO_TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" + TPC_TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" + ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES},${TRINO_TEST_MODULES},${TPC_TEST_MODULES} -am clean install -DskipTests + ./build/mvn ${MVN_OPT} -pl ${JDBC_TEST_MODULES} test + ./build/mvn ${MVN_OPT} -pl ${TRINO_TEST_MODULES} test + ./build/mvn ${MVN_OPT} -pl ${TPC_TEST_MODULES} test \ + -Dmaven.plugin.scalatest.exclude.tags='' \ + -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -336,78 +340,8 @@ jobs: path: | **/target/unit-tests.log **/kyuubi-jdbc-engine.log* - - trino-it: - name: Trino Test - runs-on: ubuntu-22.04 - strategy: - fail-fast: false - max-parallel: 1 - matrix: - java: - - 8 - comment: [ "normal" ] - steps: - - uses: actions/checkout@v4 - - name: Free up disk space - run: ./.github/scripts/free_disk_space.sh - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Setup JDK ${{ matrix.java }} - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: ${{ matrix.java }} - cache: 'maven' - check-latest: false - - name: Setup Maven - uses: ./.github/actions/setup-maven - - name: Cache Engine Archives - uses: ./.github/actions/cache-engine-archives - - name: Build and test Trino with maven w/o linters - run: | - TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test - - name: Upload test logs - if: failure() - uses: actions/upload-artifact@v3 - with: - name: unit-tests-log-java-${{ matrix.java }}-trino-${{ matrix.comment }} - path: | - **/target/unit-tests.log **/kyuubi-trino-engine.log* - tpch-tpcds: - name: TPC-H and TPC-DS Tests - runs-on: ubuntu-22.04 - env: - SPARK_LOCAL_IP: localhost - steps: - - uses: actions/checkout@v4 - - name: Free up disk space - run: ./.github/scripts/free_disk_space.sh - - name: Tune Runner VM - uses: ./.github/actions/tune-runner-vm - - name: Setup JDK 8 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: 8 - cache: 'maven' - check-latest: false - - name: Setup Maven - uses: ./.github/actions/setup-maven - - name: Cache Engine Archives - uses: ./.github/actions/cache-engine-archives - - name: Run TPC-DS Tests - run: | - TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test \ - -Dmaven.plugin.scalatest.exclude.tags='' \ - -Dtest=none -DwildcardSuites=org.apache.kyuubi.operation.tpcds,org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite,org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite - kyuubi-on-k8s-it: name: Kyuubi Server On Kubernetes Integration Test runs-on: ubuntu-22.04 diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 9669bbdcb04..4f9d1a3c27a 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -107,11 +107,6 @@ jobs: echo "you need to manually fix it by following the information told by the command above" echo "---------------------------------------------------------------------------------" - shellcheck: - name: Super Linter and Shellcheck - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v4 - name: Super Linter Checks uses: github/super-linter/slim@v5 env: