[fix][ci] Fix OWASP dep check GH actions workflow (apache#21831)

lhotari · Jan 2, 2024 · 92e0e47 · 92e0e47
1 parent c0b89eb
commit 92e0e47
Show file tree

Hide file tree

Showing 2 changed files with 69 additions and 10 deletions.
diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml
@@ -34,9 +34,10 @@ jobs:
       JOB_NAME: Check ${{ matrix.branch }}
       GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
     runs-on: ubuntu-22.04
-    timeout-minutes: 45
+    timeout-minutes: 75
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         include:
           - branch: master
@@ -63,9 +64,10 @@ jobs:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
-          key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }}
+            !~/.m2/repository/org/owasp/dependency-check-data
+          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+          lookup-only: true
           restore-keys: |
-            ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
             ${{ runner.os }}-m2-dependencies-core-modules-
 
@@ -78,19 +80,55 @@ jobs:
       - name: run install by skip tests
         run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true
 
+      - name: OWASP cache key weeknum
+        id: get-weeknum
+        run: |
+          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Restore OWASP Dependency Check data
+        id: restore-owasp-dependency-check-data
+        uses: actions/cache/restore@v3
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
+          enableCrossOsArchive: true
+          restore-keys: |
+            owasp-dependency-check-data-
+
+      - name: Update OWASP Dependency Check data
+        id: update-owasp-dependency-check-data
+        if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }}
+        run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only
+
+      - name: Save OWASP Dependency Check data
+        if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
+        uses: actions/cache/save@v3
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }}
+          enableCrossOsArchive: true
+
       - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true)
         run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true
 
-      - name: run OWASP Dependency Check for distribution/offloaders and distribution/io
-        run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io
-        if: !cancelled()
+      - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false)
+        if: ${{ !cancelled() }}
+        run: |
+          mvnprojects=$(mvn -B -ntp -Dscan=false initialize \
+            | grep -- "-< .* >-" \
+            | sed -E 's/.*-< (.*) >-.*/\1/' \
+            | grep -E 'pulsar-io-|tiered-storage-|offloader' \
+            | tr '\n' ',' | sed 's/,$/\n/' )
+          set -xe
+          mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}"
 
       - name: Upload OWASP Dependency Check reports
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: always()
         with:
           name: owasp-dependency-check-reports-${{ matrix.branch }}
           path: |
-            distribution/server/target/dependency-check-report.html
-            distribution/offloaders/target/dependency-check-report.html
-            distribution/io/target/dependency-check-report.html
+            **/target/dependency-check-report.html
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
@@ -1359,9 +1359,12 @@ jobs:
           path: |
             ~/.m2/repository/*/*/*
             !~/.m2/repository/org/apache/pulsar
+            !~/.m2/repository/org/owasp/dependency-check-data
           key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+          lookup-only: true
           restore-keys: |
             ${{ runner.os }}-m2-dependencies-core-modules-
+
       - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
         uses: actions/setup-java@v3
         with:
@@ -1378,6 +1381,24 @@ jobs:
         run: |
           cd $HOME
           $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+      - name: OWASP cache key weeknum
+        id: get-weeknum
+        run: |
+          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Restore OWASP Dependency Check data
+        id: restore-owasp-dependency-check-data
+        uses: actions/cache/restore@v3
+        timeout-minutes: 5
+        with:
+          path: ~/.m2/repository/org/owasp/dependency-check-data
+          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
+          enableCrossOsArchive: true
+          restore-keys: |
+            owasp-dependency-check-data-
+
       # Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
       - name: trigger dependency check
         run: |