build-and-push-services.yml

# Copyright 2022 Open Ag Data Alliance
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build and push OADA images

permissions: read-all

on:
  push:
  pull_request:

jobs:
  setup:
    runs-on: ubuntu-latest

    outputs:
      services: ${{ steps.services.outputs.list }}
      version: ${{ steps.ref.outputs.version }}
      release: ${{ steps.ref.outputs.release }}
      major: ${{ steps.semver.outputs.major }}
      minor: ${{ steps.semver.outputs.minor }}
      patch: ${{ steps.semver.outputs.patch }}
      prerelease: ${{ steps.semver.outputs.prerelease }}
      build: ${{ steps.semver.outputs.build }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Find Services
        id: services
        run: |
          SERVICES=$(ls -1 oada/services | tr '\n' ',' | sed 's/,/","/g' | sed 's/^/["/' | sed 's/,"$/]/')
          echo list="${SERVICES}" >> "$GITHUB_OUTPUT"

      - name: Parse Ref
        id: ref
        run: |
          echo 'Processing git ref:' $GITHUB_REF

          # Release version is just the release number
          if [[ $GITHUB_REF == refs/heads/release/* ]]; then
            VERSION=${GITHUB_REF#refs/heads/release/}
            #RELEASE=true

          elif [[ $GITHUB_REF == refs/tags/* ]]; then
            if [[ $GITHUB_REF == refs/tags/v* ]]; then
              VERSION=${GITHUB_REF#refs/tags/v}
              RELEASE=true
            else
              VERSION=tag-${GITHUB_REF#refs/tags/}
            fi

          # Branch version is branch name (with '/' -> '-')
          elif [[ $GITHUB_REF == refs/heads/* ]]; then
            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g')

            # Expect for the default_branch, which gets version "edge"
            if [ "$VERSION" == "${{ github.event.repository.default_branch }}" ]; then
              VERSION=edge
            fi

          # PR versions are pr-<github pr number>
          elif [[ $GITHUB_REF == refs/pull/* ]]; then
            VERSION=pr-${{ github.event.number }}

          else
            echo ::error ::Can not determine version of service -- unexpected job trigger? Stopping.
            exit 1
          fi

          echo version="${VERSION}" >> "$GITHUB_OUTPUT"
          echo release="${RELEASE}" >> "$GITHUB_OUTPUT"

      - name: Parse Semver
        id: semver
        if: ${{ steps.ref.outputs.release }}
        uses: booxmedialtd/ws-action-parse-semver@v1.4.7
        with:
          input_string: ${{ steps.ref.outputs.version }}
          #version_extractor_regex: '\/v(.*)$'

  prebuild:
    name: Build and cache shared Docker layers

    runs-on: ubuntu-latest

    # Only run one build at a time to fix layer reuse?
    concurrency: prebuild-${{ github.sha }}

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3.2.0

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3.7.1
        with:
          version: latest

      - name: Cache docker layers
        uses: actions/cache@v4
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}-oada
          restore-keys: |
            ${{ runner.os }}-buildx-${{ github.sha }}-
            ${{ runner.os }}-buildx-

      - name: Build base OADA images
        uses: docker/build-push-action@v6.9.0
        with:
          context: oada
          file: oada/Dockerfile
          platforms: linux/amd64
          build-args: |
            GIT_REF=${{ github.sha }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max

      - # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
        name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

  build-and-push-services:
    name: Build and Push images to DockerHub and GHCR

    needs:
      - setup
      - prebuild

    permissions:
      id-token: write
      packages: write

    strategy:
      matrix:
        service: ${{ fromJson(needs.setup.outputs.services) }}
        context:
          - oada
        include:
          - service: support/proxy
            context: support/proxy
          - service: support/proxy-http-only
            context: support/proxy
            target: http-only

    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3.2.0

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3.7.1
        with:
          version: latest

      - name: Cache docker layers
        uses: actions/cache@v4
        id: cache
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}-${{ matrix.service }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ github.sha }}-
            ${{ runner.os }}-buildx-

      - name: Prepare Images
        id: images
        run: |
          # Doesn't like upper case
          OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
          # Name on DockerHub (doesn't like extra / in images?)
          DOCKER_IMAGE=${OWNER}/$(echo ${{ matrix.service }} | tr '/' '-')
          # Name on GHCR
          GHCR_IMAGE=ghcr.io/${OWNER}/${{ matrix.service}}

          echo dockerhub="${DOCKER_IMAGE}" >> "$GITHUB_OUTPUT"
          echo ghcr="${GHCR_IMAGE}" >> "$GITHUB_OUTPUT"

      - name: Login to DockerHub
        #if: github.event_name != 'pull_request'
        uses: docker/login-action@v3.3.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GitHub Container Registry
        #if: github.event_name != 'pull_request'
        uses: docker/login-action@v3.3.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ steps.images.outputs.dockerhub }}
            ${{ steps.images.outputs.ghcr}}
          labels: |
            org.opencontainers.image.title=${{ matrix.service }}
            org.opencontainers.image.description=An OADA microservice
          # Don't update latest on prereleases?
          flavor: latest=${{ !!(needs.setup.outputs.release && !needs.setup.outputs.prerelease) }}
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{major}}
            type=schedule,pattern=nightly
            type=edge
            type=ref,event=pr
            type=sha

      - name: Build and push to images
        uses: docker/build-push-action@v6.9.0
        with:
          context: ${{ matrix.context }}
          file: ${{ matrix.context }}/Dockerfile
          target: ${{ matrix.target }}
          platforms: linux/amd64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          build-args: |
            GIT_REF=${{ github.sha }}
            OADA_SERVICE=${{ matrix.service }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max

      - # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
        name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

  # Use Snyk to check service docker images
  snyk-services:
    name: Snyk Checks

    needs:
      - setup
      - build-and-push-services

    permissions:
      security-events: write

    strategy:
      matrix:
        service: ${{ fromJson(needs.setup.outputs.services) }}
        context:
          - oada
        include:
          - service: support/proxy
            context: support/proxy

    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Find Docker tag for Snyk
        id: tag
        run: |
          # Doesn't like upper case
          OWNER=$(echo ${{ github.repository_owner }} | tr '[:upper:]' '[:lower:]')
          # Name on DockerHub (doesn't like extra / in images?)
          DOCKER_IMAGE=${OWNER}/$(echo ${{ matrix.service }} | tr '/' '-')
          # Name on GHCR
          #GHCR_IMAGE=ghcr.io/${OWNER}/${{ matrix.service}}

          TAG="${DOCKER_IMAGE}:edge"
          if [[ "${{ needs.setup.outputs.release }}" ]]; then
            if [[ "${{ needs.setup.outputs.prerelease }}" ]]; then
              TAG="${DOCKER_IMAGE}:edge"
            else
              TAG="${DOCKER_IMAGE}:latest"
            fi
          fi

          echo tag="${TAG}" >> "$GITHUB_OUTPUT"
          echo org="${OWNER}" >> "$GITHUB_OUTPUT"
          echo cur="${GHCR_IMAGE}:sha-${GITHUB_SHA::8}" >> "$GITHUB_OUTPUT"

      - name: Monitor Service image with Snyk
        uses: snyk/actions/docker@master
        # Don't break workflow on errors?
        continue-on-error: true
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          command: container monitor
          image: ${{ steps.tag.outputs.tag }}
          args: --org=${{ steps.tag.outputs.org }} --file=${{ matrix.context }}/Dockerfile

      - name: Test current Service image with Snyk
        uses: snyk/actions/docker@master
        # Don't break workflow on errors?
        continue-on-error: true
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          image: ${{ steps.tag.outputs.tag }}
          args: --org=${{ steps.tag.outputs.org }} --file=${{ matrix.context }}/Dockerfile

      - name: Replace security-severity undefined for license-related findings
        run: |
          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk.sarif
          sed -i 's/"security-severity": "null"/"security-severity": "0"/g' snyk.sarif

      - name: Secure Code Warrior
        uses: SecureCodeWarrior/github-action-add-sarif-contextual-training@v1
        with:
          inputSarifFile: ./snyk.sarif
          outputSarifFile: ./securecodewarrior.sarif
          githubToken: ${{ secrets.GITHUB_TOKEN }}

      - name: Upload SARIF file to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: ./securecodewarrior.sarif
          #sarif_file: ./snyk.sarif

  # Create a "release" compose file and attack it to a draft GitHub release.
  # TODO: Disallow release if high severity vulnerabilities, failing tests, etc.?
  create-release:
    needs:
      - setup
      - build-and-push-services

    permissions:
      contents: write
      discussions: write

    if: ${{ needs.setup.outputs.release }}

    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up yq
        uses: chrisdickinson/setup-yq@latest
        with:
          yq-version: v4.25.2

      - name: Login to DockerHub
        uses: docker/login-action@v3.3.0
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3.3.0
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Get Release Compose File
        id: release
        run: |
          RELEASE=docker-compose.release.yml

          # Use commit of release as version??
          OADA_VERSION="sha-${GITHUB_SHA::7}" \
            RELEASE_VERSION="${{ needs.setup.outputs.version }}" \
            ./release/make.sh | tee $RELEASE

          echo composefile="${RELEASE}" >> "$GITHUB_OUTPUT"

      # TODO: More detailed check?
      - name: Verify Compose File
        run: docker-compose -f ${{ steps.release.outputs.composefile }} config

      - name: Create Release
        id: create_release
        uses: softprops/action-gh-release@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ github.ref }}
          name: OADA v${{ needs.setup.outputs.version }}
          # Make draft and wait for person to release it?
          draft: true
          prerelease: ${{ !!needs.setup.outputs.prerelease }}
          token: ${{ secrets.GITHUB_TOKEN}}

      - name: Upload Release Compose File
        id: upload-release-asset
        uses: actions/upload-release-asset@v1.0.2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ steps.create_release.outputs.upload_url }}
          asset_path: ${{ steps.release.outputs.composefile }}
          asset_name: docker-compose.yml
          asset_content_type: application/yaml