[improve][build] Parameterize CI build so that Java 21 can be selecte…

…d for manual runs
lhotari · Oct 19, 2023 · e2e386b · e2e386b
1 parent ecd40e4
commit e2e386b
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 33 deletions.
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -29,7 +29,16 @@ on:
       collect_coverage:
         description: 'Collect test coverage and upload to Codecov'
         required: true
-        default: 'true'
+        type: boolean
+        default: true
+      jdk_major_version:
+        description: 'JDK major version to use for the build'
+        required: true
+        type: choice
+        options:
+          - '17'
+          - '21'
+        default: '17'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -46,25 +55,33 @@ jobs:
   preconditions:
     name: Preconditions
     runs-on: ubuntu-22.04
-    if: (github.event_name != 'schedule') || (github.repository == 'apache/pulsar')
     outputs:
       docs_only: ${{ steps.check_changes.outputs.docs_only }}
       changed_tests: ${{ steps.changes.outputs.tests_files }}
       need_owasp: ${{ steps.changes.outputs.need_owasp }}
       collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
+      jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
 
     steps:
+      - name: Select JDK major version
+        id: jdk_major_version
+        run: |
+          echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT
+
       - name: checkout
+        if: ${{ github.event_name == 'pull_request' }}
         uses: actions/checkout@v3
 
       - name: Detect changed files
+        if: ${{ github.event_name == 'pull_request' }}
         id:   changes
         uses: apache/pulsar-test-infra/paths-filter@master
         with:
           filters: .github/changes-filter.yaml
           list-files: csv
 
       - name: Check changed files
+        if: ${{ github.event_name == 'pull_request' }}
         id: check_changes
         run: |
           if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
@@ -96,6 +113,7 @@ jobs:
     env:
       JOB_NAME: Build and License check
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
@@ -125,11 +143,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Check source code license headers
         run: mvn -B -T 8 -ntp initialize apache-rat:check license:check
@@ -172,6 +190,7 @@ jobs:
       JOB_NAME: CI - Unit - ${{ matrix.name }}
       COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     runs-on: ubuntu-22.04
     timeout-minutes: ${{ matrix.timeout || 60 }}
     needs: ['preconditions', 'build-and-license-check']
@@ -233,11 +252,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK ${{ matrix.jdk || '17' }}
+      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: ${{ matrix.jdk || '17' }}
+          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -309,6 +328,8 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 30
     needs: ['preconditions', 'unit-tests']
+    env:
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
 
     steps:
@@ -337,11 +358,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK ${{ matrix.jdk || '17' }}
+      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: ${{ matrix.jdk || '17' }}
+          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -391,6 +412,8 @@ jobs:
     if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
     env:
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -417,11 +440,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -465,6 +488,7 @@ jobs:
       JOB_NAME: CI - Integration - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     strategy:
       fail-fast: false
       matrix:
@@ -497,6 +521,13 @@ jobs:
 
           - name: Shade on Java 17
             group: SHADE_RUN
+            runtime_jdk: 17
+            setup: ./build/run_integration_group.sh SHADE_BUILD
+            no_coverage: true
+
+          - name: Shade on Java 21
+            group: SHADE_RUN
+            runtime_jdk: 21
             setup: ./build/run_integration_group.sh SHADE_BUILD
             no_coverage: true
 
@@ -532,11 +563,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -622,6 +653,7 @@ jobs:
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
     env:
       PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -648,11 +680,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -731,6 +763,8 @@ jobs:
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     env:
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
+      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -763,11 +797,11 @@ jobs:
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -787,7 +821,7 @@ jobs:
           # build docker image
           # include building of Pulsar SQL, Connectors, Offloaders and server distros
           mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
-          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" \
+          -DUBUNTU_MIRROR="${UBUNTU_MIRROR}" -DUBUNTU_SECURITY_MIRROR="${UBUNTU_SECURITY_MIRROR}" -DIMAGE_JDK_MAJOR_VERSION="${IMAGE_JDK_MAJOR_VERSION}" \
           -Pmain,docker -Dmaven.test.skip=true -Ddocker.squash=true \
           -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
 
@@ -841,6 +875,7 @@ jobs:
       JOB_NAME: CI - System - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     strategy:
       fail-fast: false
       matrix:
@@ -901,11 +936,11 @@ jobs:
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -984,6 +1019,7 @@ jobs:
     if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
     env:
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
 
     steps:
       - name: checkout
@@ -1012,11 +1048,11 @@ jobs:
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -1072,6 +1108,7 @@ jobs:
       JOB_NAME: CI Flaky - System - ${{ matrix.name }}
       PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     strategy:
       fail-fast: false
       matrix:
@@ -1114,11 +1151,11 @@ jobs:
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: Install gh-actions-artifact-client.js
         uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master
@@ -1214,6 +1251,7 @@ jobs:
     if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
     env:
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -1232,11 +1270,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-all-
 
-      - name: Set up JDK 17
+      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: 17
+          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}
 
       - name: build package
         run: mvn -B clean package -DskipTests -T 1C -ntp
@@ -1249,6 +1287,7 @@ jobs:
     if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
     env:
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
     steps:
       - name: checkout
         uses: actions/checkout@v3
@@ -1274,11 +1313,11 @@ jobs:
           key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
-      - name: Set up JDK ${{ matrix.jdk || '17' }}
+      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
           distribution: 'temurin'
-          java-version: ${{ matrix.jdk || '17' }}
+          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
 
       - name: Clean Disk
         uses: ./.github/actions/clean-disk

diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile
@@ -19,6 +19,8 @@
 
 FROM ubuntu:22.04
 
+ARG JDK_MAJOR_VERSION=17
+
 # prepare the directory for pulsar related files
 RUN mkdir /pulsar
 
@@ -38,7 +40,7 @@ RUN mkdir -p /etc/apt/keyrings \
      && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
      && apt-get update \
      && apt-get -y dist-upgrade \
-     && apt-get -y install temurin-17-jdk
+     && apt-get -y install temurin-${JDK_MAJOR_VERSION:-17}-jdk
 
 # Compile and install gtest & gmock
 RUN cd /usr/src/googletest && \

diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile
@@ -54,6 +54,7 @@ FROM ubuntu:22.04
 ARG DEBIAN_FRONTEND=noninteractive
 ARG UBUNTU_MIRROR=mirror://mirrors.ubuntu.com/mirrors.txt
 ARG UBUNTU_SECURITY_MIRROR=http://security.ubuntu.com/ubuntu/
+ARG JDK_MAJOR_VERSION=17
 
 # Install some utilities
 RUN sed -i -e "s|http://archive\.ubuntu\.com/ubuntu/|${UBUNTU_MIRROR:-mirror://mirrors.ubuntu.com/mirrors.txt}|g" \
@@ -71,10 +72,10 @@ RUN mkdir -p /etc/apt/keyrings \
      && echo "deb [signed-by=/etc/apt/keyrings/adoptium.asc] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list \
      && apt-get update \
      && apt-get -y dist-upgrade \
-     && apt-get -y install temurin-17-jdk \
+     && apt-get -y install temurin-${JDK_MAJOR_VERSION:-17}-jdk \
      && export ARCH=$(uname -m | sed -r 's/aarch64/arm64/g' |  awk '!/arm64/{$0="amd64"}1') \
-     && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security \
-     && echo networkaddress.cache.negative.ttl=1 >> /usr/lib/jvm/temurin-17-jdk-$ARCH/conf/security/java.security \
+     && echo networkaddress.cache.ttl=1 >> /usr/lib/jvm/temurin-${JDK_MAJOR_VERSION:-17}-jdk-$ARCH/conf/security/java.security \
+     && echo networkaddress.cache.negative.ttl=1 >> /usr/lib/jvm/temurin-${JDK_MAJOR_VERSION:-17}-jdk-$ARCH/conf/security/java.security \
 
 # Cleanup apt
 RUN apt-get -y --purge autoremove \

diff --git a/docker/pulsar/pom.xml b/docker/pulsar/pom.xml
@@ -50,6 +50,7 @@
   <properties>
     <UBUNTU_MIRROR>mirror://mirrors.ubuntu.com/mirrors.txt</UBUNTU_MIRROR>
     <UBUNTU_SECURITY_MIRROR>http://security.ubuntu.com/ubuntu/</UBUNTU_SECURITY_MIRROR>
+    <IMAGE_JDK_MAJOR_VERSION>17</IMAGE_JDK_MAJOR_VERSION>
   </properties>
 
   <profiles>
@@ -89,6 +90,7 @@
                           <PULSAR_CLIENT_PYTHON_VERSION>${pulsar.client.python.version}</PULSAR_CLIENT_PYTHON_VERSION>
                           <UBUNTU_MIRROR>${UBUNTU_MIRROR}</UBUNTU_MIRROR>
                           <UBUNTU_SECURITY_MIRROR>${UBUNTU_SECURITY_MIRROR}</UBUNTU_SECURITY_MIRROR>
+                          <JDK_MAJOR_VERSION>${IMAGE_JDK_MAJOR_VERSION}</JDK_MAJOR_VERSION>
                         </args>
                         <contextDir>${project.basedir}</contextDir>
                         <tags>