Skip to content

Commit

Permalink
Add OPENMPI and pass QE example after moving
Browse files Browse the repository at this point in the history
  • Loading branch information
unkcpz committed Jun 15, 2024
1 parent ce3da1a commit e5f4dde
Show file tree
Hide file tree
Showing 14 changed files with 535 additions and 97 deletions.
12 changes: 12 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
version: 2
updates:
# Maintain dependencies for GitHub Actions
- package-ecosystem: github-actions
directory: /
schedule:
interval: monthly
groups:
gha-dependencies:
patterns:
- '*'
79 changes: 79 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
---
name: Build images and upload them to ghcr.io

env:
BUILDKIT_PROGRESS: plain

on:
workflow_call:
inputs:
runsOn:
description: GitHub Actions Runner image
required: true
type: string
platforms:
description: Target platforms for the build (linux/amd64 and/or linux/arm64)
required: true
type: string
outputs:
images:
description: Images identified by digests
value: ${{ jobs.build.outputs.images }}

jobs:
build:
name: ${{ inputs.platforms }}
runs-on: ${{ inputs.runsOn }}
timeout-minutes: 120

outputs:
images: ${{ steps.bake_metadata.outputs.images }}

# Make sure we fail if any command in a piped command sequence fails
defaults:
run:
shell: bash -e -o pipefail {0}

steps:

- name: Checkout Repo ⚡️
uses: actions/checkout@v4

- name: Set up QEMU
if: ${{ inputs.platforms != 'linux/amd64' }}
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3

- name: Login to GitHub Container Registry 🔑
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Build and upload to ghcr.io 📤
id: build-upload
uses: docker/bake-action@v4
with:
push: true
# Using provenance to disable default attestation so it will build only desired images:
# https://github.com/orgs/community/discussions/45969
provenance: false
set: |
*.platform=${{ inputs.platforms }}
*.output=type=registry,push-by-digest=true,name-canonical=true
*.cache-to=type=gha,scope=${{ github.workflow }},mode=max
*.cache-from=type=gha,scope=${{ github.workflow }}
files: |
docker-bake.hcl
build.json
.github/workflows/env.hcl
- name: Set output variables
id: bake_metadata
run: |
.github/workflows/extract-image-names.sh | tee -a "${GITHUB_OUTPUT}"
env:
BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }}
44 changes: 0 additions & 44 deletions .github/workflows/build_and_test.yaml

This file was deleted.

2 changes: 2 additions & 0 deletions .github/workflows/env.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# env.hcl
REGISTRY = "ghcr.io"
45 changes: 45 additions & 0 deletions .github/workflows/extract-image-names.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/bin/bash

set -euo pipefail

# Extract image names together with their sha256 digests
# from the docker/bake-action metadata output.
# These together uniquely identify newly built images.

# The input to this script is a JSON string passed via BAKE_METADATA env variable
# Here's example input (trimmed to relevant bits):
# BAKE_METADATA: {
# "qc-base": {
# "containerimage.descriptor": {
# "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
# "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
# "size": 6170,
# },
# "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
# "image.name": "ghcr.io/containers4hpc/qc-base"
# },
# "qc-full-stack": {
# "image.name": "ghcr.io/containers4hpc/qc-full-stack"
# "containerimage.digest": "sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
# "...": ""
# }
# }
#
# Example output (real output is on one line):
#
# images={
# "QC_BASE_IMAGE": "ghcr.io/containers4hpc/base@sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d",
# "QC_FULL_STACK_IMAGE": "ghcr.io/containers4hpc/full-stack@sha256:85ee91f61be1ea601591c785db038e5899d68d5fb89e07d66d9efbe8f352ee48",
# }
#
# This json output is later turned to environment variables using fromJson() GHA builtin
# (e.g. BASE_IMAGE=ghcr.io/containers4hpc/base@sha256:8e57a52b...)
# and these are in turn read in the docker-compose.<target>.yml files for tests.

if [[ -z ${BAKE_METADATA-} ]];then
echo "ERROR: Environment variable BAKE_METADATA is not set!"
exit 1
fi

images=$(echo "${BAKE_METADATA}" | jq -c '. as $base |[to_entries[] |{"key": (.key|ascii_upcase|sub("-"; "_"; "g") + "_IMAGE"), "value": [(.value."image.name"|split(",")[0]),.value."containerimage.digest"]|join("@")}] |from_entries')
echo "images=$images"
99 changes: 99 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
---
name: Docker

on:
pull_request:
paths-ignore:
- "**.md"
- ruff.toml
- bumpver.toml
- .pre-commit-config.yaml
push:
branches:
- main
tags:
- "v*"
workflow_dispatch:

# https://docs.github.com/en/actions/using-jobs/using-concurrency
concurrency:
# only cancel in-progress jobs or runs for the current workflow - matches against branch & tags
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true

env:
FORCE_COLOR: 1

jobs:

build-amd64:
uses: ./.github/workflows/build.yml
with:
runsOn: ubuntu-22.04
platforms: linux/amd64

test-amd64:
needs: build-amd64
strategy:
fail-fast: false
matrix:
target: ["qc-base", "qc-full-stack"]
uses: ./.github/workflows/test.yml
with:
runsOn: ubuntu-22.04
images: ${{ needs.build-amd64.outputs.images }}
target: ${{ matrix.target }}
integration: false

build:
needs: test-amd64
uses: ./.github/workflows/build.yml
with:
runsOn: ubuntu-22.04
platforms: linux/amd64,linux/arm64

# To save arm64 runner resources, we run the tests only on main
# and only for full-stack image (same for integration tests below).
test-arm64:
if: >-
github.repository == 'containers4hpc/quantum-container'
&& (github.ref_type == 'tag' || github.ref_name == 'main')
needs: build
uses: ./.github/workflows/test.yml
with:
runsOn: buildjet-4vcpu-ubuntu-2204-arm
images: ${{ needs.build.outputs.images }}
target: qc-full-stack
integration: false

test-integration:
name: Integration tests
needs: build
strategy:
fail-fast: false
# Trick to exclude arm64 tests from PRs
# https://github.com/orgs/community/discussions/26253
matrix:
runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm]
isPR:
- ${{ github.event_name == 'pull_request' }}
exclude:
- isPR: true
runner: buildjet-4vcpu-ubuntu-2204-arm

uses: ./.github/workflows/test.yml
with:
runsOn: ${{ matrix.runner }}
images: ${{ needs.build.outputs.images }}
target: qc-full-stack
integration: true

publish-ghcr:
needs: [build, test-amd64]
uses: ./.github/workflows/publish.yml
with:
runsOn: ubuntu-22.04
images: ${{ needs.build.outputs.images }}
registry: ghcr.io
secrets: inherit

90 changes: 90 additions & 0 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
---
name: Publish images to Docker container registries

env:
# https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables
DOCKER_METADATA_PR_HEAD_SHA: true

on:
workflow_call:
inputs:
runsOn:
description: GitHub Actions Runner image
required: true
type: string
images:
description: Images built in build step
required: true
type: string
registry:
description: Docker container registry
required: true
type: string

jobs:

release:
runs-on: ${{ inputs.runsOn }}
timeout-minutes: 30
strategy:
fail-fast: true
matrix:
target: ["qc-base", "qc-full-stack"]

steps:
- uses: actions/checkout@v4

- name: Login to GitHub Container Registry 🔑
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Login to DockerHub 🔑
uses: docker/login-action@v3
if: inputs.registry == 'docker.io'
with:
registry: docker.io
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Read build variables
id: build_vars
run: |
vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries')
echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}"
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
env: ${{ fromJSON(steps.build_vars.outputs.vars) }}
with:
# e.g. ghcr.io/containers4hpc/qc-full-stack
# type=raw,value=python-${{ env.PYTHON_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }}
images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ matrix.target }}
tags: |
type=ref,event=pr
type=edge,enable={{is_default_branch}}
type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1
- name: Determine source image
id: images
run: |
src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ matrix.target }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]')
echo "src=$src" | tee -a "${GITHUB_OUTPUT}"
- name: Push image
uses: akhilerm/[email protected]
with:
src: ${{ steps.images.outputs.src }}
dst: ${{ steps.meta.outputs.tags }}

- name: Docker Hub Description
if: inputs.registry == 'docker.io'
uses: peter-evans/dockerhub-description@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
repository: containers4hpc/${{ matrix.target }}
short-description: ${{ github.event.repository.description }}
Loading

0 comments on commit e5f4dde

Please sign in to comment.