[improve][fn] Optimize Function Worker startup by lazy loading and di… #1453
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| name: Pulsar CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - branch-2.11 | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| MAVEN_OPTS: -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 | |
| # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job | |
| # it's possible to rerun individual failed jobs when the build artifacts are available | |
| # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR | |
| ARTIFACT_RETENTION_DAYS: 3 | |
| jobs: | |
| changed_files_job: | |
| name: Preconditions | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| docs_only: ${{ steps.check_changes.outputs.docs_only }} | |
| cpp_only: ${{ steps.check_changes.outputs.cpp_only }} | |
| changed_tests: ${{ steps.changes.outputs.tests_files }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Detect changed files | |
| id: changes | |
| uses: apache/pulsar-test-infra/paths-filter@master | |
| with: | |
| filters: .github/changes-filter.yaml | |
| list-files: csv | |
| - name: Check changed files | |
| id: check_changes | |
| run: | | |
| echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" | |
| echo "::set-output name=cpp_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.cpp_count) && fromJSON(steps.changes.outputs.cpp_count) > 0 }}" | |
| - name: Check if the PR has been approved for testing | |
| if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| env: | |
| GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| run: | | |
| build/pulsar_ci_tool.sh check_ready_to_test | |
| build-and-license-check: | |
| needs: changed_files_job | |
| name: Build and License check | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Check source code license headers | |
| run: mvn -B -T 8 -ntp initialize apache-rat:check license:check | |
| - name: Build core-modules | |
| run: | | |
| mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true | |
| - name: Check binary licenses | |
| run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Save maven build results to Github artifact cache so that the results can be reused | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \ | |
| tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \ | |
| -I zstd -cf - .m2/repository/org/apache/pulsar | |
| cd $GITHUB_WORKSPACE | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \ | |
| tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| unit-tests: | |
| name: CI - Unit - ${{ matrix.name }} | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['changed_files_job', 'build-and-license-check'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Other | |
| group: OTHER | |
| timeout: 75 | |
| - name: Brokers - Broker Group 1 | |
| group: BROKER_GROUP_1 | |
| - name: Brokers - Broker Group 2 | |
| group: BROKER_GROUP_2 | |
| - name: Brokers - Broker Group 3 | |
| group: BROKER_GROUP_3 | |
| - name: Brokers - Client Api | |
| group: BROKER_CLIENT_API | |
| - name: Brokers - Client Impl | |
| group: BROKER_CLIENT_IMPL | |
| - name: Proxy | |
| group: PROXY | |
| - name: Pulsar IO | |
| group: PULSAR_IO | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ matrix.jdk || '17' }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.jdk || '17' }} | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Run setup commands | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run unit test group '${{ matrix.group }}' | |
| run: | | |
| CHANGED_TESTS="${{ needs.changed_files_job.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| with: | |
| name: Unit-${{ matrix.group }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Upload possible heap dump | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| name: Unit-${{ matrix.group }}-heapdump | |
| path: /tmp/*.hprof | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| pulsar-java-test-image: | |
| name: Build Pulsar java-test-image docker image | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: ['changed_files_job', 'build-and-license-check'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} | |
| env: | |
| UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Build java-test-image docker image | |
| run: | | |
| # build docker image | |
| mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \ | |
| -Dmaven.test.skip=true -Ddockerfile.build.squash=true -DskipSourceReleaseAssembly=true \ | |
| -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true | |
| - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| integration-tests: | |
| name: CI - Integration - ${{ matrix.name }} | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['changed_files_job', 'pulsar-java-test-image'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} | |
| env: | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Backwards Compatibility | |
| group: BACKWARDS_COMPAT | |
| - name: Cli | |
| group: CLI | |
| - name: Messaging | |
| group: MESSAGING | |
| - name: Shade on Java 8 | |
| group: SHADE_RUN | |
| runtime_jdk: 8 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| - name: Shade on Java 11 | |
| group: SHADE_RUN | |
| runtime_jdk: 11 | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| - name: Shade on Java 17 | |
| group: SHADE_RUN | |
| setup: ./build/run_integration_group.sh SHADE_BUILD | |
| - name: Standalone | |
| group: STANDALONE | |
| - name: Transaction | |
| group: TRANSACTION | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Set up runtime JDK ${{ matrix.runtime_jdk }} | |
| uses: actions/setup-java@v2 | |
| if: ${{ matrix.runtime_jdk }} | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.runtime_jdk }} | |
| - name: Run integration test group '${{ matrix.group }}' | |
| run: | | |
| ./build/run_integration_group.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| with: | |
| name: Integration-${{ matrix.group }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: Integration-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| delete-integration-test-docker-image-artifact: | |
| name: "Delete integration test docker image artifact" | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'changed_files_job', | |
| 'integration-tests' | |
| ] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete docker image from GitHub Actions Artifacts | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-java-test-image.zst | |
| cpp-tests: | |
| name: CI - CPP, Python Tests | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 120 | |
| needs: [ | |
| 'changed_files_job', | |
| 'integration-tests' | |
| ] | |
| if: always() | |
| steps: | |
| - name: check condition | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| if [[ ! ( ( \ | |
| "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \ | |
| && "${{ needs.integration-tests.result }}" == "success" \ | |
| ) || ( \ | |
| "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \ | |
| ) ) ]]; then | |
| echo "Required jobs haven't been completed successfully." | |
| exit 1 | |
| fi | |
| - name: checkout | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Cache local Maven repository | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Clean Disk | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Install gh-actions-artifact-client.js | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| cd $GITHUB_WORKSPACE | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-server-distribution | |
| - name: copy python tests | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| mvn -B -Pskip-all -ntp -pl pulsar-functions/instance package | |
| - name: build cpp artifacts | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| echo "Build C++ client library" | |
| pulsar-client-cpp/docker-build.sh | |
| - name: run c++ tests | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: pulsar-client-cpp/docker-tests.sh | |
| - name: Upload test-logs | |
| if: ${{ failure() && needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: actions/upload-artifact@v3 | |
| continue-on-error: true | |
| with: | |
| name: cpp-tests-logs | |
| path: test-logs | |
| pulsar-test-latest-version-image: | |
| name: Build Pulsar docker image | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: ['changed_files_job', 'build-and-license-check'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| env: | |
| UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/ | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Clean Disk | |
| uses: ./.github/actions/clean-disk | |
| with: | |
| mode: full | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Build latest-version-image docker image | |
| run: | | |
| # build docker image | |
| # include building of Pulsar SQL, Connectors, Offloaders and server distros | |
| mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \ | |
| -Pmain,docker -Dmaven.test.skip=true -Ddockerfile.build.squash=true \ | |
| -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true | |
| # check full build artifacts licenses | |
| - name: Check binary licenses | |
| run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz | |
| - name: Clean up disk space | |
| run: | | |
| # release disk space since saving docker image consumes local disk space | |
| # | |
| echo "::group::Available diskspace before cleaning" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| echo "::group::Clean build directory" | |
| # docker build changes some files to root ownership, fix this before deleting files | |
| sudo chown -R $USER:$GROUP . | |
| # clean build directories | |
| time git clean -fdx | |
| echo "::endgroup::" | |
| echo "::group::Available diskspace after cleaning build directory" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| echo "::group::Delete maven repository" | |
| # delete maven repository | |
| time rm -rf ~/.m2/repository | |
| echo "::endgroup::" | |
| echo "::group::Available diskspace after cleaning maven repository" | |
| time df -BM / /mnt | |
| echo "::endgroup::" | |
| - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| system-tests: | |
| name: CI - System - ${{ matrix.name }} | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: ['changed_files_job', 'pulsar-test-latest-version-image'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| env: | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Tiered FileSystem | |
| group: TIERED_FILESYSTEM | |
| - name: Tiered JCloud | |
| group: TIERED_JCLOUD | |
| - name: Function | |
| group: FUNCTION | |
| - name: Schema | |
| group: SCHEMA | |
| - name: Pulsar Connectors - Thread | |
| group: PULSAR_CONNECTORS_THREAD | |
| - name: Pulsar Connectors - Process | |
| group: PULSAR_CONNECTORS_PROCESS | |
| - name: Pulsar IO | |
| group: PULSAR_IO | |
| clean_disk: true | |
| - name: Sql | |
| group: SQL | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when needed | |
| if: ${{ matrix.clean_disk }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run system test group '${{ matrix.group }}' | |
| run: | | |
| ./build/run_integration_group.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: System-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| with: | |
| name: System-${{ matrix.name }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| flaky-system-tests: | |
| name: CI Flaky - System - ${{ matrix.name }} | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 60 | |
| needs: [ 'changed_files_job', 'pulsar-test-latest-version-image' ] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| env: | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Pulsar IO - Oracle | |
| group: PULSAR_IO_ORA | |
| clean_disk: true | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when needed | |
| if: ${{ matrix.clean_disk }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache local Maven repository | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache | |
| run: | | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run system test group '${{ matrix.group }}' | |
| run: | | |
| ./build/run_integration_group.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Publish Test Report | |
| uses: apache/pulsar-test-infra/action-junit-report@master | |
| if: ${{ always() }} | |
| with: | |
| report_paths: 'test-reports/TEST-*.xml' | |
| annotate_only: 'true' | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: System-${{ matrix.group }}-container-logs | |
| path: tests/integration/target/container-logs | |
| retention-days: 7 | |
| - name: Upload Surefire reports | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ !success() }} | |
| with: | |
| name: System-${{ matrix.name }}-surefire-reports | |
| path: surefire-reports | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| delete-system-test-docker-image-artifact: | |
| name: "Delete system test docker image artifact" | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'changed_files_job', | |
| 'system-tests', | |
| 'flaky-system-tests' | |
| ] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete docker image from GitHub Actions Artifacts | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst | |
| macos-build: | |
| name: Build Pulsar on MacOS | |
| runs-on: macos-11 | |
| timeout-minutes: 120 | |
| needs: ['changed_files_job', 'integration-tests'] | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' && needs.changed_files_job.outputs.cpp_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-all- | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: 17 | |
| - name: build package | |
| run: mvn -B clean package -DskipTests -T 1C -ntp | |
| owasp-dep-check: | |
| name: OWASP dependency check | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 120 | |
| needs: [ 'changed_files_job', 'integration-tests' ] | |
| if: ${{ needs.changed_files_job.outputs.need_owasp == 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| with: | |
| limit-access-to-actor: true | |
| - name: Cache Maven dependencies | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ~/.m2/repository/*/*/* | |
| !~/.m2/repository/org/apache/pulsar | |
| key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-m2-dependencies-core-modules- | |
| - name: Set up JDK ${{ matrix.jdk || '17' }} | |
| uses: actions/setup-java@v2 | |
| with: | |
| distribution: 'temurin' | |
| java-version: ${{ matrix.jdk || '17' }} | |
| - name: Clean Disk | |
| uses: ./.github/actions/clean-disk | |
| - name: Install gh-actions-artifact-client.js | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Restore maven build results from Github artifact cache | |
| run: | | |
| cd $HOME | |
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries | |
| # Projects dependent on flume, hdfs, hbase, and presto currently excluded from the scan. | |
| - name: trigger dependency check | |
| run: | | |
| mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \ | |
| -pl '!pulsar-sql,!distribution/server,!distribution/io,!distribution/offloaders,!pulsar-sql/presto-distribution,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb' | |
| - name: Upload report | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ cancelled() || failure() }} | |
| continue-on-error: true | |
| with: | |
| name: dependency report | |
| path: target/dependency-check-report.html | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| # This job is required for pulls to be merged. | |
| # It depends on all other jobs in this workflow. | |
| # It cleans up the binaries in the same job in order to not spin up another runner for basically doing nothing. | |
| pulsar-ci-checks-completed: | |
| name: "Pulsar CI checks completed" | |
| if: always() | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'changed_files_job', | |
| 'unit-tests', | |
| 'integration-tests', | |
| 'system-tests', | |
| 'flaky-system-tests', | |
| 'macos-build', | |
| 'cpp-tests' | |
| ] | |
| steps: | |
| - name: Check that all required jobs were completed successfully | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| if [[ ! ( ( \ | |
| "${{needs.changed_files_job.outputs.cpp_only}}" == "false" \ | |
| && "${{ needs.unit-tests.result }}" == "success" \ | |
| && "${{ needs.integration-tests.result }}" == "success" \ | |
| && "${{ needs.system-tests.result }}" == "success" \ | |
| && "${{ needs.macos-build.result }}" == "success" \ | |
| && "${{ needs.cpp-tests.result }}" == "success" \ | |
| ) || ( \ | |
| "${{needs.changed_files_job.outputs.cpp_only}}" == "true" \ | |
| && "${{ needs.system-tests.result }}" == "success" \ | |
| && "${{ needs.cpp-tests.result }}" == "success" \ | |
| ) ) ]]; then | |
| echo "Required jobs haven't been completed successfully." | |
| exit 1 | |
| fi | |
| - name: checkout | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: actions/checkout@v2 | |
| - name: Tune Runner VM | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Install gh-actions-artifact-client.js | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master | |
| - name: Delete maven repository binaries from GitHub Actions Artifacts | |
| if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }} | |
| run: | | |
| gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true | |
| gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true |