From 038d8d7015a47236699f107784bbe6c893315636 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 09:42:07 +0200 Subject: [PATCH 01/13] [fix][ci] Fix OWASP dep check GH actions workflow - Fix "Invalid workflow file" error - follow up for #21826 - GHA requires "if: ${{ !cancelled() }}" --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 8563e382a4d05..04fcebe3402e1 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -83,7 +83,7 @@ jobs: - name: run OWASP Dependency Check for distribution/offloaders and distribution/io run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io - if: !cancelled() + if: ${{ !cancelled() }} - name: Upload OWASP Dependency Check reports uses: actions/upload-artifact@v3 From 207f71deebe091a65ffd03bb9e09dd7535073a20 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 10:58:10 +0200 Subject: [PATCH 02/13] Run check for offloaders/tiered-storage and pulsar-io connectors --- .github/workflows/ci-owasp-dependency-check.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 04fcebe3402e1..52bef7eb4b2ae 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -81,9 +81,16 @@ jobs: - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - name: run OWASP Dependency Check for distribution/offloaders and distribution/io - run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/offloaders,distribution/io + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors if: ${{ !cancelled() }} + run: | + mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ + | grep -- "-< .* >-" \ + | sed -E 's/.*-< (.*) >-.*/\1/' \ + | grep -E 'pulsar-io-|tiered-storage-|offloader' \ + | tr '\n' ',' | sed 's/,$/\n/' ) + set -xe + mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports uses: actions/upload-artifact@v3 From 813d0f7c367c48dcf3633fe681065797f9cf74a6 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 11:36:16 +0200 Subject: [PATCH 03/13] add "--fail-at-end" to scan all --- .github/workflows/ci-owasp-dependency-check.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 52bef7eb4b2ae..b4daa2c5b9113 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -90,7 +90,7 @@ jobs: | grep -E 'pulsar-io-|tiered-storage-|offloader' \ | tr '\n' ',' | sed 's/,$/\n/' ) set -xe - mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl "${mvnprojects}" + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports uses: actions/upload-artifact@v3 @@ -98,6 +98,4 @@ jobs: with: name: owasp-dependency-check-reports-${{ matrix.branch }} path: | - distribution/server/target/dependency-check-report.html - distribution/offloaders/target/dependency-check-report.html - distribution/io/target/dependency-check-report.html + **/target/dependency-check-report.html \ No newline at end of file From 5dc7faa245de34d2962cd80804b8fcf5233a3a4a Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 12:48:30 +0200 Subject: [PATCH 04/13] Add caching for OWASP dependency check data --- .github/workflows/ci-owasp-dependency-check.yaml | 13 +++++++++++-- .github/workflows/pulsar-ci.yaml | 10 ++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index b4daa2c5b9113..99aeabd74245b 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -63,12 +63,21 @@ jobs: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar - key: ${{ runner.os }}-m2-dependencies-owasp-${{ hashFiles('**/pom.xml') }} + !~/.m2/repository/org/owasp/dependency-check-data + key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} + lookup-only: true restore-keys: | - ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- + - name: Cache OWASP Dependency Check data + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data + enableCrossOsArchive: true + - name: Set up JDK ${{ matrix.jdk || '17' }} uses: actions/setup-java@v3 with: diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 02496a82392c8..3b57c27a38dd1 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1359,9 +1359,19 @@ jobs: path: | ~/.m2/repository/*/*/* !~/.m2/repository/org/apache/pulsar + !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- + + - name: Cache OWASP Dependency Check data + uses: actions/cache@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data + enableCrossOsArchive: true + - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: From 229b1ddeb4e37ab710138e7d77cdab18aae35efb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 12:54:04 +0200 Subject: [PATCH 05/13] Don't fail checks for pulsar-io modules --- .github/workflows/ci-owasp-dependency-check.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 99aeabd74245b..9d68bd26e9a52 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -90,7 +90,7 @@ jobs: - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true - - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors + - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false) if: ${{ !cancelled() }} run: | mvnprojects=$(mvn -B -ntp -Dscan=false initialize \ @@ -99,7 +99,7 @@ jobs: | grep -E 'pulsar-io-|tiered-storage-|offloader' \ | tr '\n' ',' | sed 's/,$/\n/' ) set -xe - mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl "${mvnprojects}" + mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports uses: actions/upload-artifact@v3 From 73a6e2c86f1d7f67050fc0c10ed0d825904553ad Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 13:51:29 +0200 Subject: [PATCH 06/13] Use cache that caches up to one week --- .../workflows/ci-owasp-dependency-check.yaml | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 9d68bd26e9a52..9a7e29b430076 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -70,14 +70,6 @@ jobs: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} ${{ runner.os }}-m2-dependencies-core-modules- - - name: Cache OWASP Dependency Check data - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: ~/.m2/repository/org/owasp/dependency-check-data - key: owasp-dependency-check-data - enableCrossOsArchive: true - - name: Set up JDK ${{ matrix.jdk || '17' }} uses: actions/setup-java@v3 with: @@ -87,6 +79,34 @@ jobs: - name: run install by skip tests run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + + - name: Update OWASP Dependency Check data + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' }} + run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only + + - name: Save OWASP Dependency Check data + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }} + enableCrossOsArchive: true + - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true) run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true From da29ecdae20b52e5e449c6b32b8bcf03f5e2f7c5 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 13:56:51 +0200 Subject: [PATCH 07/13] Run jobs one-by-one so that cache is properly leveraged --- .github/workflows/ci-owasp-dependency-check.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 9a7e29b430076..9ff3320696449 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -37,6 +37,7 @@ jobs: timeout-minutes: 45 strategy: fail-fast: false + max-parallel: 1 matrix: include: - branch: master @@ -95,11 +96,11 @@ jobs: enableCrossOsArchive: true - name: Update OWASP Dependency Check data - if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' }} + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' && matrix.branch == 'master' }} run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only - name: Save OWASP Dependency Check data - if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' }} + if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' && matrix.branch == 'master' }} uses: actions/cache/save@v3 timeout-minutes: 5 with: From be0a171850530f3e1dcde15e5b1e4cc09f6bb316 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:01:23 +0200 Subject: [PATCH 08/13] Use actions/upload-artifact@v4 --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 9ff3320696449..990adeed7ff8e 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -123,7 +123,7 @@ jobs: mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}" - name: Upload OWASP Dependency Check reports - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() with: name: owasp-dependency-check-reports-${{ matrix.branch }} From d76c813b19c86e2a31eaf0898280b3da6dfa0bcb Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:04:12 +0200 Subject: [PATCH 09/13] Use same cache in pulsar-ci.yaml --- .github/workflows/pulsar-ci.yaml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 3b57c27a38dd1..1d6e2431aeb2b 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1361,17 +1361,10 @@ jobs: !~/.m2/repository/org/apache/pulsar !~/.m2/repository/org/owasp/dependency-check-data key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }} + lookup-only: true restore-keys: | ${{ runner.os }}-m2-dependencies-core-modules- - - name: Cache OWASP Dependency Check data - uses: actions/cache@v3 - timeout-minutes: 5 - with: - path: ~/.m2/repository/org/owasp/dependency-check-data - key: owasp-dependency-check-data - enableCrossOsArchive: true - - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} uses: actions/setup-java@v3 with: @@ -1388,6 +1381,22 @@ jobs: run: | cd $HOME $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries + + - name: OWASP cache key weeknum + id: get-weeknum + run: | + echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT + shell: bash + + - name: Restore OWASP Dependency Check data + id: restore-owasp-dependency-check-data + uses: actions/cache/restore@v3 + timeout-minutes: 5 + with: + path: ~/.m2/repository/org/owasp/dependency-check-data + key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} + enableCrossOsArchive: true + # Projects dependent on flume, hdfs, and hbase currently excluded from the scan. - name: trigger dependency check run: | From da8515efeccd5ea9b6be6af9c25230153ab36545 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:34:44 +0200 Subject: [PATCH 10/13] Increase timeout --- .github/workflows/ci-owasp-dependency-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 990adeed7ff8e..d65ad9cf9918e 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -34,7 +34,7 @@ jobs: JOB_NAME: Check ${{ matrix.branch }} GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} runs-on: ubuntu-22.04 - timeout-minutes: 45 + timeout-minutes: 75 strategy: fail-fast: false max-parallel: 1 From be6356ecb690faf09107f2ec7e947c8ba182d0e7 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:37:45 +0200 Subject: [PATCH 11/13] Restore previous key as basis --- .github/workflows/ci-owasp-dependency-check.yaml | 2 ++ .github/workflows/pulsar-ci.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index d65ad9cf9918e..fe7fda4d109c0 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -94,6 +94,8 @@ jobs: path: ~/.m2/repository/org/owasp/dependency-check-data key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- - name: Update OWASP Dependency Check data if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' && matrix.branch == 'master' }} diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml index 1d6e2431aeb2b..e339dd9948ab6 100644 --- a/.github/workflows/pulsar-ci.yaml +++ b/.github/workflows/pulsar-ci.yaml @@ -1396,6 +1396,8 @@ jobs: path: ~/.m2/repository/org/owasp/dependency-check-data key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }} enableCrossOsArchive: true + restore-keys: | + owasp-dependency-check-data- # Projects dependent on flume, hdfs, and hbase currently excluded from the scan. - name: trigger dependency check From 53d173569e2bdee14e6b4d6170e8377567938d24 Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:41:46 +0200 Subject: [PATCH 12/13] Update condition to update cache --- .github/workflows/ci-owasp-dependency-check.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index fe7fda4d109c0..45084c9483615 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -98,11 +98,11 @@ jobs: owasp-dependency-check-data- - name: Update OWASP Dependency Check data - if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' && matrix.branch == 'master' }} + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only - name: Save OWASP Dependency Check data - if: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' && matrix.branch == 'master' }} + if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} uses: actions/cache/save@v3 timeout-minutes: 5 with: From f8596d7178601e406d7c255c4ca91d7de812edcd Mon Sep 17 00:00:00 2001 From: Lari Hotari Date: Tue, 2 Jan 2024 14:45:36 +0200 Subject: [PATCH 13/13] Update cache if previous step was successful --- .github/workflows/ci-owasp-dependency-check.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-owasp-dependency-check.yaml b/.github/workflows/ci-owasp-dependency-check.yaml index 45084c9483615..ea8a3b698dcf8 100644 --- a/.github/workflows/ci-owasp-dependency-check.yaml +++ b/.github/workflows/ci-owasp-dependency-check.yaml @@ -98,11 +98,12 @@ jobs: owasp-dependency-check-data- - name: Update OWASP Dependency Check data + id: update-owasp-dependency-check-data if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only - name: Save OWASP Dependency Check data - if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }} + if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }} uses: actions/cache/save@v3 timeout-minutes: 5 with: