Skip to content

Pulsar CI

Pulsar CI #1855

Workflow file for this run

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
name: Pulsar CI
on:
pull_request:
branches:
- master
- branch-*
- pulsar-*
schedule:
# scheduled job with JDK 17
- cron: '0 12 * * *'
# scheduled job with JDK 21
# if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
- cron: '0 6 * * *'
workflow_dispatch:
inputs:
collect_coverage:
description: 'Collect test coverage and upload to Codecov'
required: true
type: boolean
default: true
jdk_major_version:
description: 'JDK major version to use for the build'
required: true
type: choice
options:
- '17'
- '21'
default: '17'
trace_test_resource_cleanup:
description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
required: true
type: choice
options:
- 'off'
- 'on'
- 'full'
default: 'off'
thread_leak_detector_wait_millis:
description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
required: true
type: number
default: 10000
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
cancel-in-progress: true
env:
MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
# defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
# it's possible to rerun individual failed jobs when the build artifacts are available
# if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
ARTIFACT_RETENTION_DAYS: 3
JDK_DISTRIBUTION: corretto
jobs:
preconditions:
name: Preconditions
runs-on: ubuntu-22.04
outputs:
docs_only: ${{ steps.check_changes.outputs.docs_only }}
changed_tests: ${{ steps.changes.outputs.tests_files }}
need_owasp: ${{ steps.changes.outputs.need_owasp }}
collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
java_non_tests: ${{ steps.changes.outputs.java_non_tests }}
steps:
- name: Cancel scheduled jobs in forks by default
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
uses: actions/github-script@v6
with:
script: |
await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
process.exit(1);
- name: Select JDK major version
id: jdk_major_version
run: |
# use JDK 21 for the scheduled build with cron expression '0 6 * * *'
if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
echo "jdk_major_version=21" >> $GITHUB_OUTPUT
exit 0
fi
# use JDK 17 for build unless overridden with workflow_dispatch input
echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT
- name: checkout
if: ${{ github.event_name == 'pull_request' }}
uses: actions/checkout@v4
- name: Detect changed files
if: ${{ github.event_name == 'pull_request' }}
id: changes
uses: apache/pulsar-test-infra/paths-filter@master
with:
filters: .github/changes-filter.yaml
list-files: csv
- name: Check changed files
if: ${{ github.event_name == 'pull_request' }}
id: check_changes
run: |
if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
else
echo docs_only=false >> $GITHUB_OUTPUT
fi
- name: Check if coverage should be collected
id: check_coverage
run: |
echo "collect_coverage=${{
(steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch'
&& (github.base_ref == 'master' || github.ref_name == 'master'))
|| (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true')
}}" >> $GITHUB_OUTPUT
- name: Check if the PR has been approved for testing
if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
env:
GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
GITHUB_TOKEN: ${{ github.token }}
run: |
build/pulsar_ci_tool.sh check_ready_to_test
build-and-license-check:
needs: preconditions
name: Build and License check
env:
JOB_NAME: Build and License check
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
runs-on: ubuntu-22.04
timeout-minutes: 60
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Check source code license headers
run: mvn -B -T 8 -ntp initialize apache-rat:check license:check
- name: Check source code style
run: mvn -B -T 8 -ntp initialize checkstyle:check
- name: Build core-modules
run: |
mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true
- name: Check binary licenses
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Save maven build results to Github artifact cache so that the results can be reused
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
-I zstd -cf - .m2/repository/org/apache/pulsar
cd $GITHUB_WORKSPACE
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
unit-tests:
name: CI - Unit - ${{ matrix.name }}
env:
JOB_NAME: CI - Unit - ${{ matrix.name }}
COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
runs-on: ubuntu-22.04
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
strategy:
fail-fast: false
matrix:
include:
- name: Other
group: OTHER
timeout: 75
- name: Brokers - Broker Group 1
group: BROKER_GROUP_1
- name: Brokers - Broker Group 2
group: BROKER_GROUP_2
- name: Brokers - Broker Group 3
group: BROKER_GROUP_3
- name: Brokers - Broker Group 4
group: BROKER_GROUP_4
- name: Brokers - Client Api
group: BROKER_CLIENT_API
- name: Brokers - Client Impl
group: BROKER_CLIENT_IMPL
- name: Proxy
group: PROXY
- name: Pulsar IO
group: PULSAR_IO
timeout: 75
- name: Pulsar IO - Elastic Search
group: PULSAR_IO_ELASTIC
- name: Pulsar IO - Kafka Connect Adaptor
group: PULSAR_IO_KAFKA_CONNECT
- name: Pulsar Client
group: CLIENT
- name: Pulsar Metadata
group: METADATA
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Clean Disk when tracing test resource cleanup
if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
uses: ./.github/actions/clean-disk
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts
- name: Run setup commands
run: |
${{ matrix.setup }}
- name: Run unit test group '${{ matrix.group }}'
run: |
CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}
- name: Upload coverage to build artifacts
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }}
- name: print JVM thread dumps when cancelled
if: cancelled()
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Report detected thread leaks
if: ${{ always() }}
run: |
if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
cd "$THREAD_LEAK_DETECTOR_DIR"
cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
fi
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
with:
name: Unit-${{ matrix.group }}-surefire-reports
path: surefire-reports
retention-days: 7
- name: Upload possible heap dump, core dump or crash files
uses: actions/upload-artifact@v4
if: ${{ always() }}
with:
name: Unit-${{ matrix.group }}-dumps
path: |
/tmp/*.hprof
**/hs_err_*.log
**/core.*
${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
retention-days: 7
if-no-files-found: ignore
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
unit-tests-upload-coverage:
name: CI - Unit - Upload Coverage
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: ['preconditions', 'unit-tests']
env:
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts
- name: Restore coverage files from build artifacts and create Jacoco reports
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true
- name: Upload Jacoco report files to build artifacts
uses: actions/upload-artifact@v4
with:
name: Jacoco-coverage-report-unittests
path: target/jacoco_test_coverage_report_unittests.zip
retention-days: 3
- name: Upload to Codecov
uses: ./.github/actions/upload-coverage
with:
flags: unittests
- name: Delete coverage files from build artifacts
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
pulsar-java-test-image:
name: Build Pulsar java-test-image docker image
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
- linux/arm64
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64
- name: Build java-test-image docker image - ${{ matrix.platform }}
run: |
# build docker image
DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
-Ddocker.platforms=${{ matrix.platform }} \
-Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
- name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
if: ${{ matrix.platform == 'linux/amd64' }}
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
integration-tests:
name: CI - Integration - ${{ matrix.name }}
runs-on: ubuntu-22.04
timeout-minutes: ${{ matrix.timeout || 60 }}
needs: ['preconditions', 'pulsar-java-test-image']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
JOB_NAME: CI - Integration - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
matrix:
include:
- name: Backwards Compatibility
group: BACKWARDS_COMPAT
no_coverage: true
- name: Cli
group: CLI
- name: Messaging
group: MESSAGING
- name: LoadBalance
group: LOADBALANCE
no_coverage: true
- name: Shade on Java 8
group: SHADE_RUN
runtime_jdk: 8
setup: ./build/run_integration_group.sh SHADE_BUILD
no_coverage: true
- name: Shade on Java 11
group: SHADE_RUN
runtime_jdk: 11
setup: ./build/run_integration_group.sh SHADE_BUILD
no_coverage: true
- name: Shade on Java 17
group: SHADE_RUN
runtime_jdk: 17
setup: ./build/run_integration_group.sh SHADE_BUILD
no_coverage: true
- name: Shade on Java 21
group: SHADE_RUN
runtime_jdk: 21
setup: ./build/run_integration_group.sh SHADE_BUILD
no_coverage: true
- name: Standalone
group: STANDALONE
- name: Transaction
group: TRANSACTION
- name: Metrics
group: METRICS
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts
- name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image
- name: Run setup commands
if: ${{ matrix.setup }}
run: |
${{ matrix.setup }}
- name: Set up runtime JDK ${{ matrix.runtime_jdk }}
uses: actions/setup-java@v4
if: ${{ matrix.runtime_jdk }}
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ matrix.runtime_jdk }}
- name: Run integration test group '${{ matrix.group }}'
run: |
if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
coverage_args="--coverage"
fi
./build/run_integration_group.sh ${{ matrix.group }} $coverage_args
- name: Upload coverage to build artifacts
if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }}
- name: print JVM thread dumps when cancelled
if: cancelled()
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
if: ${{ !success() }}
with:
name: Integration-${{ matrix.group }}-surefire-reports
path: surefire-reports
retention-days: 7
- name: Upload container logs
uses: actions/upload-artifact@v4
if: ${{ !success() }}
continue-on-error: true
with:
name: Integration-${{ matrix.group }}-container-logs
path: tests/integration/target/container-logs
retention-days: 7
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
integration-tests-upload-coverage:
name: CI - Integration - Upload Coverage
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: ['preconditions', 'integration-tests']
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
env:
PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts
- name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image
- name: Restore coverage files from build artifacts and create Jacoco reports
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true
- name: Upload Jacoco report files to build artifacts
uses: actions/upload-artifact@v4
with:
name: Jacoco-coverage-report-inttests
path: target/jacoco_test_coverage_report_inttests.zip
retention-days: 3
- name: Upload to Codecov
uses: ./.github/actions/upload-coverage
with:
flags: inttests
- name: Delete coverage files from build artifacts
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
delete-integration-test-docker-image-artifact:
name: "Delete integration test docker image artifact"
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: [
'preconditions',
'integration-tests',
'integration-tests-upload-coverage'
]
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Delete docker image from GitHub Actions Artifacts
run: |
gh-actions-artifact-client.js delete pulsar-java-test-image.zst
pulsar-test-latest-version-image:
name: Build Pulsar docker image
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'build-and-license-check']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Clean Disk
uses: ./.github/actions/clean-disk
with:
mode: full
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Build latest-version-image docker image
run: |
# build docker image
# include building of Connectors, Offloaders and server distros
DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
-Pmain,docker -Dmaven.test.skip=true \
-Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
# check full build artifacts licenses
- name: Check binary licenses
run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz
- name: Run Trivy container scan
uses: aquasecurity/trivy-action@master
if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
with:
image-ref: "apachepulsar/pulsar:latest"
scanners: vuln
severity: CRITICAL,HIGH,MEDIUM,LOW
limit-severities-for-sarif: true
format: 'sarif'
output: 'trivy-results.sarif'
- name: Upload Trivy scan results to GitHub Security tab
uses: github/codeql-action/upload-sarif@v3
if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
with:
sarif_file: 'trivy-results.sarif'
- name: Clean up disk space
run: |
# release disk space since saving docker image consumes local disk space
#
echo "::group::Available diskspace before cleaning"
time df -BM / /mnt
echo "::endgroup::"
echo "::group::Clean build directory"
# docker build changes some files to root ownership, fix this before deleting files
sudo chown -R $USER:$GROUP .
# clean build directories
time git clean -fdx
echo "::endgroup::"
echo "::group::Available diskspace after cleaning build directory"
time df -BM / /mnt
echo "::endgroup::"
echo "::group::Delete maven repository"
# delete maven repository
time rm -rf ~/.m2/repository
echo "::endgroup::"
echo "::group::Available diskspace after cleaning maven repository"
time df -BM / /mnt
echo "::endgroup::"
- name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
system-tests:
name: CI - System - ${{ matrix.name }}
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'pulsar-test-latest-version-image']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
JOB_NAME: CI - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
matrix:
include:
- name: Tiered FileSystem
group: TIERED_FILESYSTEM
- name: Tiered JCloud
group: TIERED_JCLOUD
- name: Function
group: FUNCTION
- name: Schema
group: SCHEMA
- name: Pulsar Connectors - Thread
group: PULSAR_CONNECTORS_THREAD
- name: Pulsar Connectors - Process
group: PULSAR_CONNECTORS_PROCESS
- name: Pulsar IO
group: PULSAR_IO
clean_disk: true
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Clean Disk when needed
if: ${{ matrix.clean_disk }}
uses: ./.github/actions/clean-disk
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts
- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
- name: Run setup commands
if: ${{ matrix.setup }}
run: |
${{ matrix.setup }}
- name: Run system test group '${{ matrix.group }}'
run: |
if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
coverage_args="--coverage"
fi
./build/run_integration_group.sh ${{ matrix.group }} $coverage_args
- name: Upload coverage to build artifacts
if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }}
- name: print JVM thread dumps when cancelled
if: cancelled()
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload container logs
uses: actions/upload-artifact@v4
if: ${{ !success() }}
continue-on-error: true
with:
name: System-${{ matrix.group }}-container-logs
path: tests/integration/target/container-logs
retention-days: 7
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
if: ${{ !success() }}
with:
name: System-${{ matrix.name }}-surefire-reports
path: surefire-reports
retention-days: 7
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
system-tests-upload-coverage:
name: CI - System - Upload Coverage
runs-on: ubuntu-22.04
timeout-minutes: 30
needs: ['preconditions', 'system-tests']
if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
env:
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
- name: Restore coverage files from build artifacts and create Jacoco reports
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
cd $GITHUB_WORKSPACE/target
zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true
- name: Upload Jacoco report files to build artifacts
uses: actions/upload-artifact@v4
with:
name: Jacoco-coverage-report-systests
path: target/jacoco_test_coverage_report_systests.zip
retention-days: 3
- name: Upload to Codecov
uses: ./.github/actions/upload-coverage
with:
flags: systests
- name: Delete coverage files from build artifacts
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
flaky-system-tests:
name: CI Flaky - System - ${{ matrix.name }}
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
JOB_NAME: CI Flaky - System - ${{ matrix.name }}
PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
strategy:
fail-fast: false
matrix:
include:
- name: Plugin
group: PLUGIN
- name: Pulsar IO - Oracle
group: PULSAR_IO_ORA
clean_disk: true
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Clean Disk when needed
if: ${{ matrix.clean_disk }}
uses: ./.github/actions/clean-disk
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
run: |
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
- name: Run setup commands
if: ${{ matrix.setup }}
run: |
${{ matrix.setup }}
- name: Run system test group '${{ matrix.group }}'
run: |
./build/run_integration_group.sh ${{ matrix.group }}
- name: print JVM thread dumps when cancelled
if: cancelled()
run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
- name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
if: ${{ always() }}
uses: ./.github/actions/copy-test-reports
- name: Publish Test Report
uses: apache/pulsar-test-infra/action-junit-report@master
if: ${{ always() }}
with:
report_paths: 'test-reports/TEST-*.xml'
annotate_only: 'true'
- name: Upload container logs
uses: actions/upload-artifact@v4
if: ${{ !success() }}
continue-on-error: true
with:
name: System-${{ matrix.group }}-container-logs
path: tests/integration/target/container-logs
retention-days: 7
- name: Upload Surefire reports
uses: actions/upload-artifact@v4
if: ${{ !success() }}
with:
name: System-${{ matrix.name }}-surefire-reports
path: surefire-reports
retention-days: 7
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
delete-system-test-docker-image-artifact:
name: "Delete system test docker image artifact"
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: [
'preconditions',
'system-tests',
'system-tests-upload-coverage',
'flaky-system-tests'
]
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Delete docker image from GitHub Actions Artifacts
run: |
gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst
macos-build:
name: Build Pulsar on MacOS
runs-on: macos-latest
timeout-minutes: 120
needs: ['preconditions', 'integration-tests']
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Cache Maven dependencies
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-all-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: build package
run: mvn -B clean package -DskipTests -T 1C -ntp
codeql:
name: Run CodeQL Analysis
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: ['preconditions', 'unit-tests']
if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
permissions:
actions: read
contents: read
security-events: write
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
CODEQL_LANGUAGE: java-kotlin
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Clean Disk when needed
if: ${{ matrix.clean_disk }}
uses: ./.github/actions/clean-disk
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Cache local Maven repository
uses: actions/cache@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ env.CODEQL_LANGUAGE }}
- name: Build Java code
run: |
mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{ env.CODEQL_LANGUAGE }}"
owasp-dep-check:
name: OWASP dependency check
runs-on: ubuntu-22.04
timeout-minutes: 120
needs: [ 'preconditions', 'integration-tests' ]
if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
env:
DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
NIST_NVD_API_KEY: ${{ secrets.NIST_NVD_API_KEY }}
steps:
- name: checkout
uses: actions/checkout@v4
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- name: Setup ssh access to build runner VM
# ssh access is enabled for builds in own forks
if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
uses: ./.github/actions/ssh-access
continue-on-error: true
with:
limit-access-to-actor: true
- name: Restore Maven repository cache
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: |
~/.m2/repository/*/*/*
!~/.m2/repository/org/apache/pulsar
key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-m2-dependencies-core-modules-
- name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
uses: actions/setup-java@v4
with:
distribution: ${{ env.JDK_DISTRIBUTION }}
java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
- name: Clean Disk
uses: ./.github/actions/clean-disk
- name: Install gh-actions-artifact-client.js
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Restore maven build results from Github artifact cache
run: |
cd $HOME
$GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
- name: OWASP cache key weeknum
id: get-weeknum
run: |
echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
shell: bash
- name: Restore OWASP Dependency Check data
id: restore-owasp-dependency-check-data
uses: actions/cache/restore@v4
timeout-minutes: 5
with:
path: ~/.m2/repository/org/owasp/dependency-check-data
key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
enableCrossOsArchive: true
restore-keys: |
owasp-dependency-check-data-
# Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
- name: trigger dependency check
run: |
mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
-pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'
- name: Upload report
uses: actions/upload-artifact@v4
if: ${{ cancelled() || failure() }}
continue-on-error: true
with:
name: dependency report
path: target/dependency-check-report.html
- name: Wait for ssh connection when build fails
# ssh access is enabled for builds in own forks
uses: ./.github/actions/ssh-access
if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
continue-on-error: true
with:
action: wait
# This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
# protected_branches section for master branch required_status_checks.
# It depends on all other jobs in this workflow.
# This job also cleans up the binaries at the end of the workflow.
pulsar-ci-checks-completed:
name: "Pulsar CI checks completed"
# run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
# this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
# jobs are not enabled for other than apache/pulsar repository.
if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: [
'preconditions',
'unit-tests',
'integration-tests',
'system-tests',
'flaky-system-tests',
'macos-build',
'unit-tests-upload-coverage',
'integration-tests-upload-coverage',
'system-tests-upload-coverage',
'owasp-dep-check',
'codeql'
]
steps:
- name: Check that all required jobs were completed successfully
if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true' }}
run: |
if [[ ! ( \
"${{ needs.preconditions.result }}" == "success" \
&& "${{ needs.unit-tests.result }}" == "success" \
&& "${{ needs.integration-tests.result }}" == "success" \
&& "${{ needs.system-tests.result }}" == "success" \
&& "${{ needs.macos-build.result }}" == "success" \
&& ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \
) ]]; then
echo "Required jobs haven't been completed successfully."
exit 1
fi
- name: checkout
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
uses: actions/checkout@v4
- name: Tune Runner VM
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
uses: ./.github/actions/tune-runner-vm
- name: Install gh-actions-artifact-client.js
if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
- name: Delete maven repository binaries from GitHub Actions Artifacts
if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
run: |
gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true
gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true