Fix / CI improvements for cloud storage (#469) #1453
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compliance | |
| on: | |
| # Run compliance jobs for pull requests, to make sure there are no issues before merging | |
| pull_request: | |
| # Re-run compliance jobs in main, to make sure there are no issues from the merge | |
| push: | |
| branches: | |
| - main | |
| # Use the release:publish event to generate compliance reports for releases | |
| # This lines up with the packaging workflow, so reports will be published when packages are published | |
| release: | |
| types: | |
| - published | |
| # Allow manual triggering of the compliance jobs | |
| workflow_dispatch: | |
| # Use latest supported language versions for compliance jobs | |
| env: | |
| JAVA_VERSION: "21" | |
| JAVA_DISTRIBUTION: "zulu" | |
| PYTHON_VERSION: "3.12" | |
| NODE_VERSION: "22" | |
| jobs: | |
| platform_compliance: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: ${{ env.JAVA_DISTRIBUTION }} | |
| java-version: ${{ env.JAVA_VERSION }} | |
| cache: gradle | |
| - name: License check | |
| run: ./gradlew checkLicense | |
| # Use a cache to save downloading the whole NVD on every build | |
| # Dependency check will automatically look for updates and download the delta | |
| - name: NVD restore cache (Java Platform) | |
| id: cacheRestore | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: nvd-cache-java-platform | |
| path: ./build/compliance-cache/nvd_java_platform | |
| - name: OWASP recreate cache | |
| if: steps.cacheRestore.outputs.cache-hit != 'true' | |
| env: | |
| NVD_API_KEY: ${{ secrets.NVD_API_KEY }} | |
| # In case of problems with this job, try enabling / disabling the remote cache | |
| NVD_DATAFEED: "https://dependency-check.github.io/DependencyCheck_Builder/nvd_cache/" | |
| run: ./gradlew dependencyCheckUpdate | |
| - name: OWASP dependency check | |
| env: | |
| NVD_API_KEY: ${{ secrets.NVD_API_KEY }} | |
| # In case of problems with this job, try enabling / disabling the remote cache | |
| NVD_DATAFEED: "https://dependency-check.github.io/DependencyCheck_Builder/nvd_cache/" | |
| run: ./gradlew dependencyCheckAggregate | |
| - name: NVD save cache (Java Platform) | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: nvd-cache-java-platform | |
| path: ./build/compliance-cache/nvd_java_platform | |
| # Always save the reports - they are especially needed when the checks have failed! | |
| - name: Store compliance reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: compliance-reports-java | |
| path: build/compliance/** | |
| python_runtime_compliance: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: PIP Upgrade | |
| run: python -m pip install --upgrade pip | |
| # Make sure to check compliance on both core and plugin dependencies | |
| - name: Install dependencies | |
| run: | | |
| cd tracdap-runtime/python | |
| pip install -r requirements.txt | |
| pip install -r requirements_plugins.txt | |
| - name: License check | |
| run: | | |
| # Source license excpetions | |
| . dev/compliance/license-config-python.sh | |
| mkdir -p build/compliance/python-runtime-licenses | |
| cd tracdap-runtime/python | |
| pip-licenses --format=json --ignore-packages $IGNORE_LICENSE > ../../build/compliance/python-runtime-licenses/python-runtime-licenses.json | |
| pip-licenses --format=html --ignore-packages $IGNORE_LICENSE > ../../build/compliance/python-runtime-licenses/python-runtime-licenses.html | |
| pip-licenses --allow-only="$ALLOWED_LICENSES" --ignore-packages $IGNORE_LICENSE | |
| # CVE-2019-8341 (safety id: 70612) is a vulnerability in the latest version of Jinja2, no fix is available | |
| # It is only pulled in as a dependency of safety check itself, it is not a dependency of the dist wheels | |
| - name: Safety check | |
| run: | | |
| mkdir -p build/compliance/python-runtime-safety | |
| cd tracdap-runtime/python | |
| safety check --ignore 70612 --output text > ../../build/compliance/python-runtime-safety/python-runtime-safety-report.txt | |
| safety check --ignore 70612 --output json > ../../build/compliance/python-runtime-safety/python-runtime-safety-report.json | |
| # Always save the reports - they are especially needed when the checks have failed! | |
| - name: Store compliance reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: compliance-reports-python | |
| path: build/compliance/** | |
| web_api_compliance: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| cache: npm | |
| cache-dependency-path: tracdap-api/packages/web/package-lock.json | |
| - name: Install dependencies | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm install | |
| - name: License check | |
| run: | | |
| cd tracdap-api/packages/web | |
| npm run compliance-licenses | |
| - name: NPM audit | |
| run: | | |
| mkdir -p build/compliance/web-api-npm-audit | |
| cd tracdap-api/packages/web | |
| npm run compliance-audit | |
| # Use a cache to save downloading global compliance data on every build (e.g. NVD) | |
| # Compliance tasks will automatically look for updates and download deltas | |
| - name: NVD restore cache (Web API) | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: nvd-cache-web-api | |
| path: ./build/compliance-cache/nvd_web_api | |
| # ODC version must be forced because the NodeJS wrapper breaks for versions >= 10.0.0 | |
| - name: OWASP dependency check | |
| env: | |
| NVD_DATAFEED: "https://dependency-check.github.io/DependencyCheck_Builder/nvd_cache/" | |
| NVD_API_KEY: ${{ secrets.NVD_API_KEY }} | |
| run: | | |
| mkdir -p build/compliance | |
| cd tracdap-api/packages/web | |
| mkdir -p ./dependency-check-bin/latest | |
| if [ -n "${NVD_API_KEY}" ]; then | |
| npm run compliance-owasp -- --nvdDatafeed "${NVD_DATAFEED}" --nvdApiKey "${NVD_API_KEY}" | |
| else | |
| npm run compliance-owasp -- --nvdDatafeed "${NVD_DATAFEED}" | |
| fi | |
| - name: NVD save cache (Web API) | |
| if: always() | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: nvd-cache-web-api | |
| path: ./build/compliance-cache/nvd_web_api | |
| # Always save the reports - they are especially needed when the checks have failed! | |
| - name: Store compliance reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: compliance-reports-web | |
| path: build/compliance/** | |
| publish_to_github: | |
| if: ${{ github.event_name == 'release' && github.event.action == 'published' }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - platform_compliance | |
| - python_runtime_compliance | |
| - web_api_compliance | |
| steps: | |
| # fetch-depth = 0 is needed to get tags for version info | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get TRAC version | |
| id: tracdap-version | |
| run: | | |
| tracdap_version=`dev/version.sh` | |
| echo "tracdap_version = ${tracdap_version}" | |
| echo "tracdap_version=${tracdap_version}" >> $GITHUB_OUTPUT | |
| - name: Fetch Java compliance report artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: compliance-reports-java | |
| path: tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }} | |
| - name: Fetch Python compliance report artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: compliance-reports-python | |
| path: tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }} | |
| - name: Fetch web API compliance report artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: compliance-reports-web | |
| path: tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }} | |
| - name: Build compliance reports tarball | |
| run: zip -r tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }}.zip tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }}/ | |
| - name: Publish compliance reports | |
| env: | |
| RELEASE_TAG: ${{ github.event.release.tag_name }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh release upload $RELEASE_TAG tracdap-compliance-reports-${{ steps.tracdap-version.outputs.tracdap_version }}.zip |