diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml deleted file mode 100644 index 9717126ea28..00000000000 --- a/.github/workflows/backend-integration-test.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Backend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - integration-test-common/** - - meta/** - - server/** - - server-common/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - test-mode: [ embedded, deploy ] - backend: [ jdbcBackend, kvBackend] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Backend Integration Test - id: integrationTest - run: > - ./gradlew test --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -P${{ matrix.backend }} -PskipWebITs - -x :web:test -x :clients:client-python:test -x :spark-connector:test -x :spark-connector:spark-common:test - -x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test - -x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }}-${{ matrix.backend }} - path: | - build/reports - integration-test/build/*.log - integration-test/build/*.tar - integration-test/build/trino-ci-container-log/hive/*.* - integration-test/build/trino-ci-container-log/hdfs/*.* - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar - distribution/**/*.log diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f339bb5bbac..93f9ec9af91 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,9 +85,6 @@ jobs: java-version: ${{ matrix.java-version }} distribution: 'temurin' - - name: Test publish to local - run: ./gradlew publishToMavenLocal -x test -PjdkVersion=${{ matrix.java-version }} - - name: Build with Gradle run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml deleted file mode 100644 index dfd192b36cd..00000000000 --- a/.github/workflows/frontend-integration-test.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Frontend Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - integration-test/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - trino-connector/** - - web/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - run: | - ./gradlew build -x test -PjdkVersion=${{ matrix.java-version }} - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Frontend Integration Test - id: integrationTest - run: | - ./gradlew --rerun-tasks -PskipTests -PtestMode=embedded -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" - ./gradlew --rerun-tasks -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} :integration-test:test --tests "com.datastrato.gravitino.integration.test.web.ui.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: integrate-test-reports-${{ matrix.java-version }} - path: | - build/reports - integration-test/build/integration-test-integration-test.log - integration-test/build/*.tar - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log - catalogs/**/*.tar \ No newline at end of file diff --git a/.github/workflows/python-integration-test.yml b/.github/workflows/python-integration-test.yml deleted file mode 100644 index 3840bfc3f0d..00000000000 --- a/.github/workflows/python-integration-test.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Python Client Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - clients/client-python/** - - common/** - - conf/** - - core/** - - meta/** - - server/** - - server-common/** - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8 ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Python Client Integration Test - id: integrationTest - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - for pythonVersion in "3.8" "3.9" "3.10" "3.11" - do - echo "Use Python version ${pythonVersion} to test the Python client." - ./gradlew -PjdkVersion=${{ matrix.java-version }} -PpythonVersion=${pythonVersion} :clients:client-python:test - # Clean Gravitino database to clean test data - rm -rf ./distribution/package/data - done - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ failure() && steps.integrationTest.outcome == 'failure' }} - with: - name: integrate test reports - path: | - build/reports - integration-test/build/integration-test.log - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log - catalogs/**/*.log \ No newline at end of file diff --git a/.github/workflows/spark-integration-test.yml b/.github/workflows/spark-integration-test.yml deleted file mode 100644 index 39bf66c5876..00000000000 --- a/.github/workflows/spark-integration-test.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Spark Integration Test - -# Controls when the workflow will run -on: - # Triggers the workflow on push or pull request events but only for the "main" branch - push: - branches: [ "main", "branch-*" ] - pull_request: - branches: [ "main", "branch-*" ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - filters: | - source_changes: - - .github/** - - api/** - - bin/** - - catalogs/** - - clients/client-java/** - - clients/client-java-runtime/** - - clients/filesystem-hadoop3/** - - clients/filesystem-hadoop3-runtime/** - - common/** - - conf/** - - core/** - - dev/** - - gradle/** - - meta/** - - server/** - - server-common/** - - spark-connector/** - - docs/open-api/** - - build.gradle.kts - - gradle.properties - - gradlew - - setting.gradle.kts - outputs: - source_changes: ${{ steps.filter.outputs.source_changes }} - - # Integration test for AMD64 architecture - test-amd64-arch: - needs: changes - if: needs.changes.outputs.source_changes == 'true' - runs-on: ubuntu-latest - timeout-minutes: 90 - strategy: - matrix: - architecture: [linux/amd64] - java-version: [ 8, 11, 17 ] - scala-version: [ 2.12 ] - test-mode: [ embedded, deploy ] - env: - PLATFORM: ${{ matrix.architecture }} - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-java@v3 - with: - java-version: ${{ matrix.java-version }} - distribution: 'temurin' - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Check required command - run: | - dev/ci/check_commands.sh - - - name: Package Gravitino - if : ${{ matrix.test-mode == 'deploy' }} - run: | - ./gradlew compileDistribution -x test -PjdkVersion=${{ matrix.java-version }} - - - name: Setup debug Github Action - if: ${{ contains(github.event.pull_request.labels.*.name, 'debug action') }} - uses: csexton/debugger-action@master - - - name: Free up disk space - run: | - dev/ci/util_free_space.sh - - - name: Spark Integration Test - id: integrationTest - run: | - if [ "${{ matrix.scala-version }}" == "2.12" ];then - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.3:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" - fi - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.4:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" - ./gradlew --rerun-tasks -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PscalaVersion=${{ matrix.scala-version }} :spark-connector:spark-3.5:test --tests "com.datastrato.gravitino.spark.connector.integration.test.**" - - - name: Upload integrate tests reports - uses: actions/upload-artifact@v3 - if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }} - with: - name: spark-connector-integrate-test-reports-${{ matrix.java-version }}-${{ matrix.test-mode }} - path: | - build/reports - spark-connector/v3.3/spark/build/spark-3.3-integration-test.log - spark-connector/v3.4/spark/build/spark-3.4-integration-test.log - spark-connector/v3.5/spark/build/spark-3.5-integration-test.log - distribution/package/logs/gravitino-server.out - distribution/package/logs/gravitino-server.log diff --git a/build.gradle.kts b/build.gradle.kts index bf1f766144c..45a8f605d62 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -410,7 +410,9 @@ subprojects { jvmArgs = listOf("-Xmx2G") useJUnitPlatform() jvmArgs(project.property("extraJvmArgs") as List<*>) - finalizedBy(tasks.getByName("jacocoTestReport")) + if (project.name != "integration-test-common") { + finalizedBy(tasks.getByName("jacocoTestReport")) + } } } diff --git a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java index a05bb3662ee..9a1093362c5 100644 --- a/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java +++ b/core/src/main/java/com/datastrato/gravitino/GravitinoEnv.java @@ -359,6 +359,7 @@ public void shutdown() { if (eventListenerManager != null) { eventListenerManager.stop(); } + LOG.info(""); LOG.info("Gravitino Environment is shut down."); }