Configurable s3 endpoint for download #2231
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build PR container | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - ".github/integration/**" | |
| - ".github/workflows/**" | |
| - ".gitignore" | |
| - "**/*.md" | |
| - ".github/dependabot.yaml" | |
| - "charts/**" | |
| - "Makefile" | |
| - "sda-admin/**" | |
| env: | |
| PR_NUMBER: ${{ github.event.number }} | |
| jobs: | |
| build_go_images: | |
| name: Build PR image (golang) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for sda-download | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./sda-download | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-download | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-download | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for sensitive-data-archive | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./sda | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }} | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }} | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| build_server_images: | |
| name: Build PR image (servers) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for postgres | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./postgresql | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-postgres | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for rabbitmq | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./rabbitmq | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-rabbitmq | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Run Trivy vulnerability scanner on postgres | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_SKIP_DB_UPDATE: true | |
| TRIVY_SKIP_JAVA_DB_UPDATE: true | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'postgres-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'postgres-results.sarif' | |
| category: postgres | |
| - name: Run Trivy vulnerability scanner on rabbitmq | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_SKIP_DB_UPDATE: true | |
| TRIVY_SKIP_JAVA_DB_UPDATE: true | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'rabbitmq-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'rabbitmq-results.sarif' | |
| category: rabbitmq | |
| build_java_images: | |
| name: Build PR image (java) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for sda-sftp-inbox | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: ./sda-sftp-inbox | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-sftp-inbox | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Run Trivy vulnerability scanner on sftp-inbox | |
| uses: aquasecurity/[email protected] | |
| env: | |
| TRIVY_SKIP_DB_UPDATE: true | |
| TRIVY_SKIP_JAVA_DB_UPDATE: true | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'inbox-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: 'inbox-results.sarif' | |
| category: sftp-inbox | |
| rabbitmq: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| name: rabbitmq-federation-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test rabbitmq federation | |
| run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test | |
| postgres: | |
| needs: | |
| - build_server_images | |
| name: postgres-test-suite | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test postgres | |
| run: docker compose -f .github/integration/postgres.yml run tests | |
| sda: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| storage: ["posix", "s3"] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test sensitive-data-archive | |
| run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test | |
| sda-sync: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: dorny/paths-filter@v3 | |
| id: changes | |
| with: | |
| filters: | | |
| sync: | |
| - "sda/cmd/sync/*" | |
| sync-api: | |
| - "sda/cmd/syncapi/*" | |
| - name: Test sda-sync | |
| run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test | |
| if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true' | |
| chart: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| - build_java_images | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| version: ["1.30", "1.31"] | |
| tls: ["true", "false"] | |
| storage: ["posix", "s3"] | |
| exclude: | |
| - version: "1.30" | |
| tls: "false" | |
| storage: "posix" | |
| - version: "1.31" | |
| tls: "false" | |
| storage: "posix" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Helm | |
| uses: azure/setup-helm@v4 | |
| - name: Initialise k3d | |
| id: initK3D | |
| run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} | |
| shell: bash | |
| - name: debug | |
| if: steps.initK3D.outcome == 'failure' | |
| run: k3d version list k3s | grep ${{matrix.version}} | |
| shell: bash | |
| - name: Deploy external services | |
| run: bash .github/integration/scripts/charts/dependencies.sh | |
| shell: bash | |
| - name: Deploy DB | |
| id: deployDB | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} ${{matrix.tls}} | |
| - name: debug | |
| if: failure() && steps.deployDB.outcome == 'failure' | |
| run: | | |
| kubectl describe pod postgres-sda-db-0 | |
| sleep 1 | |
| kubectl logs postgres-sda-db-0 | |
| - name: Deploy MQ | |
| id: deployMQ | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} | |
| shell: bash | |
| - name: debug | |
| if: failure() && steps.deployMQ.outcome == 'failure' | |
| run: | | |
| kubectl describe pod broker-sda-mq-0 | |
| sleep 1 | |
| kubectl logs broker-sda-mq-0 | |
| - name: Deploy pipeline | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} | |
| shell: bash | |
| - name: Check deployment | |
| run: | | |
| sleep 30 | |
| for n in api auth download finalize inbox ingest mapper reencrypt sync syncapi verify; do | |
| if [ ${{matrix.storage}} == "posix" ] && [ "$n" == "auth" ] || [ "$n" == "sync" ] || [ "$n" == "syncapi" ]; then | |
| continue | |
| fi | |
| if [ ! $(kubectl get pods -l role="$n" -o=jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep true) ]; then | |
| echo "$n is not ready after 30s, exiting" | |
| exit 1 | |
| fi | |
| done | |
| - name: test | |
| if: always() | |
| run: | | |
| kubectl get pods | |
| sleep 1 | |
| for svc in api auth finalize inbox ingest mapper reencrypt sync syncapi verify; do | |
| echo "## describe $svc" && kubectl describe pod -l role="$svc" | |
| sleep 1 | |
| echo "## logs $svc" && kubectl logs -l role="$svc" | |
| sleep 1 | |
| done | |
| shell: bash |