From fc8c4cf3097e5cdfa3c0cd6815a73f2c4bfd039c Mon Sep 17 00:00:00 2001 From: yuqi Date: Tue, 3 Sep 2024 17:35:24 +0800 Subject: [PATCH] Fix --- .../workflows/backend-integration-test.yml | 384 +++++++++--------- .github/workflows/build.yml | 7 +- .github/workflows/flink-integration-test.yml | 208 +++++----- .../workflows/frontend-integration-test.yml | 214 +++++----- .github/workflows/python-integration-test.yml | 172 ++++---- .github/workflows/spark-integration-test.yml | 220 +++++----- 6 files changed, 601 insertions(+), 604 deletions(-) diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 1c6596f772a..cd84ad4cfa3 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -1,192 +1,192 @@ -name: Backend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - integration-test-common/** - - iceberg/** - - meta/** - - server/** - - server-common/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - test-on-push: - needs: changes - if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - # Integration test for AMD64 architecture - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log - - test-on-pr: - needs: changes - if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - # Integration test for AMD64 architecture - architecture: [ linux/amd64 ] - java-version: [ 17 ] - test-mode: [ embedded, deploy ] - include: - - test-mode: 'embedded' - backend: 'h2' - - test-mode: 'deploy' - backend: 'mysql' - - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if: ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) - id: integrationTest - run: > - ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false - -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - -x :authorizations:authorization-ranger:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - iceberg/iceberg-rest-server/build/*.log - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log - distribution/package/logs/*.out - distribution/package/logs/*.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log \ No newline at end of file +#name: Backend Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - integration-test/** +# - integration-test-common/** +# - iceberg/** +# - meta/** +# - server/** +# - server-common/** +# - trino-connector/** +# - web/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# test-on-push: +# needs: changes +# if: (github.event_name == 'push' && needs.changes.outputs.source_changes == 'true') +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# # Integration test for AMD64 architecture +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# test-mode: [ embedded, deploy ] +# include: +# - test-mode: 'embedded' +# backend: 'h2' +# - test-mode: 'deploy' +# backend: 'mysql' +# +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if : ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) +# id: integrationTest +# run: > +# ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false +# -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test +# -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test +# -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test +# -x :authorizations:authorization-ranger:test +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} +# path: | +# build/reports +# iceberg/iceberg-rest-server/build/*.log +# integration-test/build/*.log +# integration-test/build/*.tar +# integration-test/build/trino-ci-container-log +# distribution/package/logs/*.out +# distribution/package/logs/*.log +# catalogs/**/*.log +# catalogs/**/*.tar +# distribution/**/*.log +# +# test-on-pr: +# needs: changes +# if: (github.event_name == 'pull_request' && needs.changes.outputs.source_changes == 'true') +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# # Integration test for AMD64 architecture +# architecture: [ linux/amd64 ] +# java-version: [ 17 ] +# test-mode: [ embedded, deploy ] +# include: +# - test-mode: 'embedded' +# backend: 'h2' +# - test-mode: 'deploy' +# backend: 'mysql' +# +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if: ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Backend Integration Test (JDK${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }}) +# id: integrationTest +# run: > +# ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PjdbcBackend=${{ matrix.backend }} -PskipWebITs -PskipDockerTests=false +# -x :web:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:test -x :spark-connector:spark-common:test +# -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test +# -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test +# -x :authorizations:authorization-ranger:test +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} +# path: | +# build/reports +# iceberg/iceberg-rest-server/build/*.log +# integration-test/build/*.log +# integration-test/build/*.tar +# integration-test/build/trino-ci-container-log +# distribution/package/logs/*.out +# distribution/package/logs/*.log +# catalogs/**/*.log +# catalogs/**/*.tar +# distribution/**/*.log \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdcf427e76b..99e2e0b5758 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -144,13 +144,10 @@ jobs: - name: Jacoco Report to PR id: jacoco - uses: madrapps/jacoco-report@fd4800e8a81e21bdf373438e5918b975df041d15 + uses: madrapps/jacoco-report@1.6 with: paths: | - ${{ github.workspace }}/api/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/common/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/server/build/reports/jacoco/test/jacocoTestReport.xml, - ${{ github.workspace }}/core/build/reports/jacoco/test/jacocoTestReport.xml + ${{ github.workspace }}/**/build/reports/jacoco/test/jacocoTestReport.xml token: ${{ secrets.GITHUB_TOKEN }} min-coverage-overall: 40 min-coverage-changed-files: 60 diff --git a/.github/workflows/flink-integration-test.yml b/.github/workflows/flink-integration-test.yml index 6e1227664cb..997dc7bb3de 100644 --- a/.github/workflows/flink-integration-test.yml +++ b/.github/workflows/flink-integration-test.yml @@ -1,104 +1,104 @@ -name: Flink Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - meta/** - - server/** - - server-common/** - - flink-connector/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Flink Integration Test - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: flink-connector-integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - flink-connector/flink/build/*.log - flink-connector/flink/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar +#name: Flink Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - meta/** +# - server/** +# - server-common/** +# - flink-connector/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Flink Integration Test +# id: integrationTest +# run: | +# ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" +# ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: flink-connector-integrate-test-reports-${{ matrix.java-version }} +# path: | +# build/reports +# flink-connector/flink/build/*.log +# flink-connector/flink/build/*.tar +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 2d730f392ff..28a669230b3 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -1,107 +1,107 @@ -name: Frontend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Frontend Integration Test - id: integrationTest - run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - integration-test/build/integration-test-integration-test.log - integration-test/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file +#name: Frontend Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - integration-test/** +# - meta/** +# - server/** +# - server-common/** +# - spark-connector/** +# - trino-connector/** +# - web/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 60 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Frontend Integration Test +# id: integrationTest +# run: | +# ./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" +# ./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :integration-test:test --tests "org.apache.gravitino.integration.test.web.ui.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: integrate-test-reports-${{ matrix.java-version }} +# path: | +# build/reports +# integration-test/build/integration-test-integration-test.log +# integration-test/build/*.tar +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml index 4e9f96bc690..abd85c901f1 100644 --- a/.github/workflows/python-integration-test.yml +++ b/.github/workflows/python-integration-test.yml @@ -1,86 +1,86 @@ -name: Python Client Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - catalogs/catalog-hadoop/** - - clients/client-python/** - - common/** - - conf/** - - core/** - - meta/** - - server/** - - server-common/** - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Python Client Integration Test - id: integrationTest - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - for pythonVersion in "3.8" "3.9" "3.10" "3.11" - do - echo "Use Python version ${pythonVersion} to test the Python client." - ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} -PskipDockerTests=false :clients:client-python:test - # Clean Gravitino database to clean test data - rm -rf ./distribution/package/data - done - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} - with: - name: integrate test reports - path: | - build/reports - integration-test/build/integration-test.log - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file +#name: Python Client Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - catalogs/catalog-hadoop/** +# - clients/client-python/** +# - common/** +# - conf/** +# - core/** +# - meta/** +# - server/** +# - server-common/** +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 30 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8 ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Python Client Integration Test +# id: integrationTest +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# for pythonVersion in "3.8" "3.9" "3.10" "3.11" +# do +# echo "Use Python version ${pythonVersion} to test the Python client." +# ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} -PskipDockerTests=false :clients:client-python:test +# # Clean Gravitino database to clean test data +# rm -rf ./distribution/package/data +# done +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} +# with: +# name: integrate test reports +# path: | +# build/reports +# integration-test/build/integration-test.log +# distribution/package/logs/gravitino-server.out +# distribution/package/logs/gravitino-server.log +# catalogs/**/*.log +# catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml index a2f11061cda..b59261b2e10 100644 --- a/.github/workflows/spark-integration-test.yml +++ b/.github/workflows/spark-integration-test.yml @@ -1,110 +1,110 @@ -name: Spark Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - iceberg/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - scala-version: [ 2.12 ] - test-mode: [ embedded, deploy ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v4 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - cache: 'gradle' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Spark Integration Test - id: integrationTest - run: | - if [ "${{ matrix.scala-version }}" == "2.12" ];then - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - fi - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} - path: | - build/reports - spark-connector/v3.3/spark/build/spark-3.3-integration-test.log - spark-connector/v3.4/spark/build/spark-3.4-integration-test.log - spark-connector/v3.5/spark/build/spark-3.5-integration-test.log - distribution/package/logs/*.out - distribution/package/logs/*.log +#name: Spark Integration Test +# +## Controls when the workflow will run +#on: +# # Triggers the workflow on push or pull request events but only for the "main" branch +# push: +# branches: [ "main", "branch-*" ] +# pull_request: +# branches: [ "main", "branch-*" ] +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# changes: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: dorny/paths-filter@v2 +# id: filter +# with: +# filters: | +# source_changes: +# - .github/** +# - api/** +# - bin/** +# - catalogs/** +# - clients/client-java/** +# - clients/client-java-runtime/** +# - clients/filesystem-hadoop3/** +# - clients/filesystem-hadoop3-runtime/** +# - common/** +# - conf/** +# - core/** +# - dev/** +# - gradle/** +# - iceberg/** +# - meta/** +# - server/** +# - server-common/** +# - spark-connector/** +# - docs/open-api/** +# - build.gradle.kts +# - gradle.properties +# - gradlew +# - setting.gradle.kts +# outputs: +# source_changes: ${{ steps.filter.outputs.source_changes }} +# +# # Integration test for AMD64 architecture +# test-amd64-arch: +# needs: changes +# if: needs.changes.outputs.source_changes == 'true' +# runs-on: ubuntu-latest +# timeout-minutes: 90 +# strategy: +# matrix: +# architecture: [linux/amd64] +# java-version: [ 8, 11, 17 ] +# scala-version: [ 2.12 ] +# test-mode: [ embedded, deploy ] +# env: +# PLATFORM: ${{ matrix.architecture }} +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions/setup-java@v4 +# with: +# java-version: ${{ matrix.java-version }} +# distribution: 'temurin' +# cache: 'gradle' +# +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v2 +# +# - name: Check required command +# run: | +# dev/ci/check_commands.sh +# +# - name: Package Gravitino +# if : ${{ matrix.test-mode == 'deploy' }} +# run: | +# ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} +# +# - name: Free up disk space +# run: | +# dev/ci/util_free_space.sh +# +# - name: Spark Integration Test +# id: integrationTest +# run: | +# if [ "${{ matrix.scala-version }}" == "2.12" ];then +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.3:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# fi +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.4:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# ./gradlew -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} -PskipDockerTests=false :spark-connector:spark-3.5:test --tests "org.apache.gravitino.spark.connector.integration.test.**" +# +# - name: Upload integrate tests reports +# uses: actions/upload-artifact@v3 +# if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} +# with: +# name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} +# path: | +# build/reports +# spark-connector/v3.3/spark/build/spark-3.3-integration-test.log +# spark-connector/v3.4/spark/build/spark-3.4-integration-test.log +# spark-connector/v3.5/spark/build/spark-3.5-integration-test.log +# distribution/package/logs/*.out +# distribution/package/logs/*.log