[FEGA usecase] Merge internal components from sda-pipeline into sda
#540
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build PR container | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - ".github/**" | |
| - ".gitignore" | |
| - "**/README.md" | |
| env: | |
| PR_NUMBER: ${{ github.event.number }} | |
| jobs: | |
| build_go_images: | |
| name: Build PR image (golang) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for sda-auth | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./sda-auth | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-auth | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-auth | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for sda-download | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./sda-download | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-download | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-download | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for sda-pipeline | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./sda-pipeline | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-pipeline | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-pipeline | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for sensitive-data-archive | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./sda | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }} | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }} | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| build_server_images: | |
| name: Build PR image (servers) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for postgres | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./postgresql | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-postgres | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Build container for rabbitmq | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./rabbitmq | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-rabbitmq | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Run Trivy vulnerability scanner on postgres | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'postgres-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'postgres-results.sarif' | |
| category: postgres | |
| - name: Run Trivy vulnerability scanner on rabbitmq | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'rabbitmq-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'rabbitmq-results.sarif' | |
| category: rabbitmq | |
| build_java_images: | |
| name: Build PR image (java) | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| packages: write | |
| security-events: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Log in to the Github Container registry | |
| uses: docker/login-action@v2 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build container for sda-sftp-inbox | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./sda-sftp-inbox | |
| push: true | |
| tags: | | |
| ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox | |
| ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-sftp-inbox | |
| labels: | | |
| org.opencontainers.image.source=${{ github.event.repository.clone_url }} | |
| org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') | |
| org.opencontainers.image.revision=${{ github.sha }} | |
| - name: Run Trivy vulnerability scanner on sftp-inbox | |
| uses: aquasecurity/[email protected] | |
| with: | |
| image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox | |
| format: "sarif" | |
| hide-progress: true | |
| ignore-unfixed: true | |
| output: 'inbox-results.sarif' | |
| severity: "CRITICAL,HIGH" | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'inbox-results.sarif' | |
| category: sftp-inbox | |
| rabbitmq: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| name: rabbitmq-federation-test | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test rabbitmq federation | |
| run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test | |
| postgres: | |
| needs: | |
| - build_server_images | |
| name: postgres-test-suite | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test postgres | |
| run: docker compose -f .github/integration/postgres.yml run tests | |
| sda: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Test sensitive-data-archive | |
| run: docker compose -f .github/integration/sda-integration.yml run integration_test | |
| chart: | |
| needs: | |
| - build_go_images | |
| - build_server_images | |
| - build_java_images | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| version: ["1.26", "1.27"] | |
| tls: ["true", "false"] | |
| storage: ["posix", "s3"] | |
| exclude: | |
| - version: "1.26" | |
| tls: "false" | |
| storage: "posix" | |
| - version: "1.27" | |
| tls: "false" | |
| storage: "posix" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Helm | |
| uses: azure/[email protected] | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Initialise k3d | |
| run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} | |
| shell: bash | |
| - name: debug | |
| if: failure() | |
| run: k3d version list k3s | grep ${{matrix.version}} | |
| shell: bash | |
| - name: Deploy external services | |
| run: bash .github/integration/scripts/charts/dependencies.sh | |
| shell: bash | |
| - name: Deploy DB | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} ${{matrix.tls}} | |
| - name: Deploy MQ | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} | |
| shell: bash | |
| - name: Deploy pipeline | |
| run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} | |
| shell: bash | |
| - name: Check deployment | |
| run: | | |
| sleep 30 | |
| for n in download finalize inbox ingest mapper verify; do | |
| if [ ! $(kubectl get pods -l role="$n" -o=jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep true) ]; then | |
| echo "$n is not ready after 30s, exiting" | |
| exit 1 | |
| fi | |
| done | |
| - name: test | |
| if: always() | |
| run: | | |
| kubectl get secret broker-sda-mq -o json | |
| kubectl get secret pipeline-sda-svc-mapper -o json | |
| kubectl get pods | |
| echo "describe mapper" && kubectl describe pod -l role=mapper | |
| sleep 1 | |
| echo "logs mapper" && kubectl logs -l role=mapper | |
| sleep 1 | |
| echo "describe broker" && kubectl logs -l role=broker | |
| shell: bash |