From 6dbd74fbcc895fd2c832abbf57d77e6d4c627793 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Wed, 19 Jun 2024 00:49:01 +0200 Subject: [PATCH] Use CI that generic for single code build repo --- .github/dependabot.yml | 12 ++ .github/workflows/build.yml | 78 ++++++++++++ .github/workflows/ci.yml | 158 ------------------------ .github/workflows/env.hcl | 2 - .github/workflows/extract-image-name.sh | 8 +- .github/workflows/main.yml | 94 ++++++++++++++ .github/workflows/publish.yml | 72 +++++++++++ .github/workflows/test.yml | 55 +++++++++ build.json | 14 +-- docker-bake.hcl | 24 +--- 10 files changed, 322 insertions(+), 195 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/env.hcl create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ce0d77c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +--- +version: 2 +updates: +# Maintain dependencies for GitHub Actions + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + groups: + gha-dependencies: + patterns: + - '*' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..ba40344 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,78 @@ +--- +name: Build images and upload them to ghcr.io + +env: + BUILDKIT_PROGRESS: plain + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + image: + description: Image identified by digests + value: ${{ jobs.build.outputs.image }} + +jobs: + build: + name: ${{ inputs.platforms }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 120 + + outputs: + image: ${{ steps.bake_metadata.outputs.image }} + + # Make sure we fail if any command in a piped command sequence fails + defaults: + run: + shell: bash -e -o pipefail {0} + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and upload to ghcr.io 📤 + id: build-upload + uses: docker/bake-action@v4 + with: + push: true + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 + provenance: false + set: | + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + *.cache-to=type=gha,scope=${{ github.workflow }},mode=max + *.cache-from=type=gha,scope=${{ github.workflow }} + files: | + docker-bake.hcl + build.json + + - name: Set output variables + id: bake_metadata + run: | + .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}" + env: + BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 864d9d7..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -name: Build images and run tests and publish - -on: - pull_request: - push: - branches: - - main - - support/** - tags: - - "v*" - workflow_dispatch: - -env: - BUILDKIT_PROGRESS: plain - FORCE_COLOR: 1 - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - build: - - runs-on: ubuntu-latest - timeout-minutes: 30 - - outputs: - image: ${{ steps.bake_metadata.outputs.image }} - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Set up QEMU - if: ${{ inputs.platforms != 'linux/amd64' }} - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - - uses: crazy-max/ghaction-github-runtime@v3 - - name: Build and upload to ghcr.io 📤 - id: build-upload - uses: docker/bake-action@v4 - with: - push: true - # Using provenance to disable default attestation so it will build only desired images: - # https://github.com/orgs/community/discussions/45969 - provenance: false - set: | - *.platform=linux/amd64 - *.output=type=registry,name-canonical=true,push-by-digest=true - *.cache-from=type=gha - *.cache-to=type=gha,mode=max - - files: | - docker-bake.hcl - build.json - .github/workflows/env.hcl - - - name: Set output variables - id: bake_metadata - run: | - .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}" - env: - BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} - - test: - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: build - - steps: - - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Run container checking libraries exist in the container - run: | - docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which pw.x" >> /tmp/cat-output.txt - docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which ph.x" >> /tmp/cat-output.txt - if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/pw.x"; then - echo "pw.x found" - else - echo "pw.x not found" - exit 1 - fi - - if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/ph.x"; then - echo "ph.x found" - else - echo "ph.x not found" - exit 1 - fi - - publish: - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [build] - if: >- - github.repository == 'containers4hpc/quantum-espresso' - && (github.ref_type == 'tag' || github.ref_name == 'main') - - steps: - - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Read build variables - id: build_vars - run: | - vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') - echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - env: ${{ fromJSON(steps.build_vars.outputs.vars) }} - with: - images: ghcr.io/${{ github.repository_owner }}/quantum-espresso - tags: | - type=edge,enable={{is_default_branch}} - type=raw,value={{tag}},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} - type=raw,value=qe-${{ env.QE_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} - type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 - - - name: Push tags - uses: akhilerm/tag-push-action@v2.2.0 - with: - src: ${{ needs.build.outputs.image }} - dst: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/env.hcl b/.github/workflows/env.hcl deleted file mode 100644 index 9a8782a..0000000 --- a/.github/workflows/env.hcl +++ /dev/null @@ -1,2 +0,0 @@ -# env.hcl -REGISTRY = "ghcr.io" diff --git a/.github/workflows/extract-image-name.sh b/.github/workflows/extract-image-name.sh index ebb56ee..a0210cc 100755 --- a/.github/workflows/extract-image-name.sh +++ b/.github/workflows/extract-image-name.sh @@ -5,24 +5,24 @@ # The input to this script is a JSON string passed via BAKE_METADATA env variable # Here's example input (trimmed to relevant bits): # BAKE_METADATA: { -# "base": { +# "python": { # "containerimage.descriptor": { # "mediaType": "application/vnd.docker.distribution.manifest.v2+json", # "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", # "size": 6170, # }, # "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", -# "image.name": "ghcr.io/pspgen/quantum-espresso" +# "image.name": "ghcr.io/cnts4sci/python" # } # } # # Example output (real output is on one line): # -# image="ghcr.io/pspgen/quantum-espresso@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12" +# image="ghcr.io/cnts4sci/python@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12" # } # # This json output is later turned to environment variables using fromJson() GHA builtin -# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/pspgen/quantum-espresso@sha256:8e57a52b...) +# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/cnts4sci/python@sha256:8e57a52b...) # and these are in turn read in the docker-compose..yml files for tests. if [[ -z ${BAKE_METADATA-} ]];then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..21cc5f2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,94 @@ +--- +name: Docker + +on: + pull_request: + paths-ignore: + - "**.md" + - ruff.toml + - bumpver.toml + - .pre-commit-config.yaml + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + + build-amd64: + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64 + + test-amd64: + needs: build-amd64 + strategy: + fail-fast: false + uses: ./.github/workflows/test.yml + with: + runsOn: ubuntu-22.04 + image: ${{ needs.build-amd64.outputs.image }} + integration: false + + build: + needs: test-amd64 + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64,linux/arm64 + + # To save arm64 runner resources, we run the tests only on main + # and only for full-stack image (same for integration tests below). + test-arm64: + if: >- + github.repository == 'cnts4sci/build-machine' + && (github.ref_type == 'tag' || github.ref_name == 'main') + needs: build + uses: ./.github/workflows/test.yml + with: + runsOn: buildjet-4vcpu-ubuntu-2204-arm + image: ${{ needs.build.outputs.image }} + integration: false + + test-integration: + name: Integration tests + needs: build + strategy: + fail-fast: false + # Trick to exclude arm64 tests from PRs + # https://github.com/orgs/community/discussions/26253 + matrix: + runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm] + isPR: + - ${{ github.event_name == 'pull_request' }} + exclude: + - isPR: true + runner: buildjet-4vcpu-ubuntu-2204-arm + + uses: ./.github/workflows/test.yml + with: + runsOn: ${{ matrix.runner }} + image: ${{ needs.build.outputs.image }} + integration: true + + publish-ghcr: + needs: [build, test-amd64] + uses: ./.github/workflows/publish.yml + with: + runsOn: ubuntu-22.04 + image: ${{ needs.build.outputs.image }} + registry: ghcr.io + secrets: inherit + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..a9e9039 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,72 @@ +--- +name: Publish image to Docker container registries + +env: + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true + REPO_NAME: ${GITHUB_REPOSITORY##*/} + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + image: + description: Image built in build step + required: true + type: string + registry: + description: Docker container registry + required: true + type: string + +jobs: + + release: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 30 + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} + with: + # e.g. ghcr.io/cnts4sci/python + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ env.REPO_NAME }} + tags: | + type=edge,enable={{is_default_branch}} + type=match,pattern=(\d{4}\.\d{4}(-.+)?),group=1 + type=raw,value=v${{ env.VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + + - name: Determine source image + id: image + run: | + src=$(echo '${{ inputs.image }}'| jq -cr '.[("${{ env.REPO_NAME }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" + + - name: Push image + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.image.outputs.src }} + dst: ${{ steps.meta.outputs.tags }} + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7b3ed34 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,55 @@ +--- +name: Test newly built image + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + image: + description: Image built in build step + required: true + type: string + integration: + description: Run integration tests + required: false + type: boolean + +jobs: + + test: + name: ${{ inputs.integration && inputs.runsOn }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 20 + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run container checking libraries exist in the container + run: | + docker run --rm ${{ inputs.image) }} /bin/bash -c "which pw.x" >> /tmp/cat-output.txt + docker run --rm ${{ inputs.image }} /bin/bash -c "which ph.x" >> /tmp/cat-output.txt + if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/pw.x"; then + echo "pw.x found" + else + echo "pw.x not found" + exit 1 + fi + + if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/ph.x"; then + echo "ph.x found" + else + echo "ph.x not found" + exit 1 + fi diff --git a/build.json b/build.json index 29bc6fa..f0a2c53 100644 --- a/build.json +++ b/build.json @@ -1,18 +1,6 @@ { "variable": { - "BUILD_BASE_IMAGE": { - "default": "ghcr.io/cnts4sci/bm:edge" - }, - "RUNTIME_BASE_IMAGE": { - "default": "docker.io/phusion/baseimage:focal-1.2.0" - }, - "OPENMPI_BUILDER": { - "default": "ghcr.io/cnts4sci/bm-openmpi:edge" - }, - "LAPACK_BUILDER": { - "default": "ghcr.io/cnts4sci/bm-lapack:edge" - }, - "QE_VERSION": { + "VERSION": { "default": "7.0" } } diff --git a/docker-bake.hcl b/docker-bake.hcl index fb7925a..5ebe75e 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,19 +6,7 @@ variable "ORGANIZATION" { default = "cnts4sci" } -variable "BULID_BASE_IMAGE" { -} - -variable "RUNTIME_BASE_IMAGE" { -} - -variable "OPENMPI_BUILDER" { -} - -variable "LAPACK_BUILDER" { -} - -variable "QE_VERSION" { +variable "VERSION" { } variable "REGISTRY" { @@ -40,13 +28,13 @@ target "qe" { inherits = ["qe-meta"] context = "." contexts = { - build-base-image = "docker-image://${BUILD_BASE_IMAGE}" - runtime-base-image = "docker-image://${RUNTIME_BASE_IMAGE}" - openmpi-builder-image = "docker-image://${OPENMPI_BUILDER}" - lapack-builder-image = "docker-image://${LAPACK_BUILDER}" + build-base-image = "docker-image://ghcr.io/cnts4sci/bm:2024.1001" + runtime-base-image = "docker-image://docker.io/phusion/baseimage:focal-1.2.0" + openmpi-builder-image = "docker-image://ghcr.io/cnts4sci/bm-openmpi:v4.1.6" + lapack-builder-image = "docker-image://ghcr.io/cnts4sci/bm-lapack:v3.10.1" } args = { - "QE_VERSION" = "${QE_VERSION}" + "QE_VERSION" = "${VERSION}" } }