diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 8563e382a4d05..ea8a3b698dcf8 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -34,9 +34,10 @@ jobs: JOB_NAME: Check ${{ matrix.branch }} GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master @@ -63,9 +64,10 @@ jobs: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + !~/.m2/repository/org/owasp/dependency-check-data + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + lookup-only: true restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- @@ -78,19 +80,55 @@ jobs: - name: run install by skip tests run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + + - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} + uses: actions/cache/save@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true + - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - name: run OWASP Dependency Check for distribution/offloaders and distribution/io - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io - if: !cancelled() + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) + if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 02496a82392c8..e339dd9948ab6 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1359,9 +1359,12 @@ jobs: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar + !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- + - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: @@ -1378,6 +1381,24 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- + # Projects dependent on flume, hdfs, and hbase currently excluded from the scan. - name: trigger dependency check run: |