From 4ddde7e19770bb02fd91fd7070f474d24d8c1d62 Mon Sep 17 00:00:00 2001 From: Jusong Yu Date: Wed, 19 Jun 2024 00:36:04 +0200 Subject: [PATCH] CI fill --- .github/dependabot.yml | 12 ++ .github/workflows/build.yml | 78 ++++++++++++ .github/workflows/ci.yml | 158 ------------------------ .github/workflows/env.hcl | 2 - .github/workflows/extract-image-name.sh | 8 +- .github/workflows/main.yml | 94 ++++++++++++++ .github/workflows/publish.yml | 72 +++++++++++ .github/workflows/test.yml | 55 +++++++++ Dockerfile | 32 +---- README.md | 24 +++- build.json | 6 - docker-bake.hcl | 10 +- 12 files changed, 343 insertions(+), 208 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/env.hcl create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..ce0d77c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +--- +version: 2 +updates: +# Maintain dependencies for GitHub Actions + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + groups: + gha-dependencies: + patterns: + - '*' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..3d3173b --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,78 @@ +--- +name: Build images and upload them to ghcr.io + +env: + BUILDKIT_PROGRESS: plain + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + platforms: + description: Target platforms for the build (linux/amd64 and/or linux/arm64) + required: true + type: string + outputs: + images: + description: Images identified by digests + value: ${{ jobs.build.outputs.images }} + +jobs: + build: + name: ${{ inputs.platforms }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 120 + + outputs: + image: ${{ steps.bake_metadata.outputs.image }} + + # Make sure we fail if any command in a piped command sequence fails + defaults: + run: + shell: bash -e -o pipefail {0} + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Set up QEMU + if: ${{ inputs.platforms != 'linux/amd64' }} + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and upload to ghcr.io 📤 + id: build-upload + uses: docker/bake-action@v4 + with: + push: true + # Using provenance to disable default attestation so it will build only desired images: + # https://github.com/orgs/community/discussions/45969 + provenance: false + set: | + *.platform=${{ inputs.platforms }} + *.output=type=registry,push-by-digest=true,name-canonical=true + *.cache-to=type=gha,scope=${{ github.workflow }},mode=max + *.cache-from=type=gha,scope=${{ github.workflow }} + files: | + docker-bake.hcl + build.json + + - name: Set output variables + id: bake_metadata + run: | + .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}" + env: + BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 864d9d7..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -name: Build images and run tests and publish - -on: - pull_request: - push: - branches: - - main - - support/** - tags: - - "v*" - workflow_dispatch: - -env: - BUILDKIT_PROGRESS: plain - FORCE_COLOR: 1 - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - - build: - - runs-on: ubuntu-latest - timeout-minutes: 30 - - outputs: - image: ${{ steps.bake_metadata.outputs.image }} - - steps: - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Set up QEMU - if: ${{ inputs.platforms != 'linux/amd64' }} - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - - uses: crazy-max/ghaction-github-runtime@v3 - - name: Build and upload to ghcr.io 📤 - id: build-upload - uses: docker/bake-action@v4 - with: - push: true - # Using provenance to disable default attestation so it will build only desired images: - # https://github.com/orgs/community/discussions/45969 - provenance: false - set: | - *.platform=linux/amd64 - *.output=type=registry,name-canonical=true,push-by-digest=true - *.cache-from=type=gha - *.cache-to=type=gha,mode=max - - files: | - docker-bake.hcl - build.json - .github/workflows/env.hcl - - - name: Set output variables - id: bake_metadata - run: | - .github/workflows/extract-image-name.sh | tee -a "${GITHUB_OUTPUT}" - env: - BAKE_METADATA: ${{ steps.build-upload.outputs.metadata }} - - test: - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: build - - steps: - - - name: Checkout Repo ⚡️ - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Run container checking libraries exist in the container - run: | - docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which pw.x" >> /tmp/cat-output.txt - docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which ph.x" >> /tmp/cat-output.txt - if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/pw.x"; then - echo "pw.x found" - else - echo "pw.x not found" - exit 1 - fi - - if cat /tmp/cat-output.txt | grep -q "/usr/local/bin/ph.x"; then - echo "ph.x found" - else - echo "ph.x not found" - exit 1 - fi - - publish: - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [build] - if: >- - github.repository == 'containers4hpc/quantum-espresso' - && (github.ref_type == 'tag' || github.ref_name == 'main') - - steps: - - uses: actions/checkout@v4 - - - name: Login to GitHub Container Registry 🔑 - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Read build variables - id: build_vars - run: | - vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') - echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - env: ${{ fromJSON(steps.build_vars.outputs.vars) }} - with: - images: ghcr.io/${{ github.repository_owner }}/quantum-espresso - tags: | - type=edge,enable={{is_default_branch}} - type=raw,value={{tag}},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} - type=raw,value=qe-${{ env.QE_VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} - type=match,pattern=v(\d{4}\.\d{4}(-.+)?),group=1 - - - name: Push tags - uses: akhilerm/tag-push-action@v2.2.0 - with: - src: ${{ needs.build.outputs.image }} - dst: ${{ steps.meta.outputs.tags }} diff --git a/.github/workflows/env.hcl b/.github/workflows/env.hcl deleted file mode 100644 index 9a8782a..0000000 --- a/.github/workflows/env.hcl +++ /dev/null @@ -1,2 +0,0 @@ -# env.hcl -REGISTRY = "ghcr.io" diff --git a/.github/workflows/extract-image-name.sh b/.github/workflows/extract-image-name.sh index ebb56ee..a0210cc 100755 --- a/.github/workflows/extract-image-name.sh +++ b/.github/workflows/extract-image-name.sh @@ -5,24 +5,24 @@ # The input to this script is a JSON string passed via BAKE_METADATA env variable # Here's example input (trimmed to relevant bits): # BAKE_METADATA: { -# "base": { +# "python": { # "containerimage.descriptor": { # "mediaType": "application/vnd.docker.distribution.manifest.v2+json", # "digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", # "size": 6170, # }, # "containerimage.digest": "sha256:8e57a52b924b67567314b8ed3c968859cad99ea13521e60bbef40457e16f391d", -# "image.name": "ghcr.io/pspgen/quantum-espresso" +# "image.name": "ghcr.io/cnts4sci/python" # } # } # # Example output (real output is on one line): # -# image="ghcr.io/pspgen/quantum-espresso@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12" +# image="ghcr.io/cnts4sci/python@sha256:79a0f984b9e03b733304fda809ad3e8eec8416992ff334052d75da00cadb8f12" # } # # This json output is later turned to environment variables using fromJson() GHA builtin -# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/pspgen/quantum-espresso@sha256:8e57a52b...) +# (e.g. BUILD_MACHINE_IMAGE=ghcr.io/cnts4sci/python@sha256:8e57a52b...) # and these are in turn read in the docker-compose..yml files for tests. if [[ -z ${BAKE_METADATA-} ]];then diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..0994cf6 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,94 @@ +--- +name: Docker + +on: + pull_request: + paths-ignore: + - "**.md" + - ruff.toml + - bumpver.toml + - .pre-commit-config.yaml + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + + build-amd64: + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64 + + test-amd64: + needs: build-amd64 + strategy: + fail-fast: false + uses: ./.github/workflows/test.yml + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build-amd64.outputs.images }} + integration: false + + build: + needs: test-amd64 + uses: ./.github/workflows/build.yml + with: + runsOn: ubuntu-22.04 + platforms: linux/amd64,linux/arm64 + + # To save arm64 runner resources, we run the tests only on main + # and only for full-stack image (same for integration tests below). + test-arm64: + if: >- + github.repository == 'cnts4sci/build-machine' + && (github.ref_type == 'tag' || github.ref_name == 'main') + needs: build + uses: ./.github/workflows/test.yml + with: + runsOn: buildjet-4vcpu-ubuntu-2204-arm + images: ${{ needs.build.outputs.images }} + integration: false + + test-integration: + name: Integration tests + needs: build + strategy: + fail-fast: false + # Trick to exclude arm64 tests from PRs + # https://github.com/orgs/community/discussions/26253 + matrix: + runner: [ubuntu-22.04, buildjet-4vcpu-ubuntu-2204-arm] + isPR: + - ${{ github.event_name == 'pull_request' }} + exclude: + - isPR: true + runner: buildjet-4vcpu-ubuntu-2204-arm + + uses: ./.github/workflows/test.yml + with: + runsOn: ${{ matrix.runner }} + images: ${{ needs.build.outputs.images }} + integration: true + + publish-ghcr: + needs: [build, test-amd64] + uses: ./.github/workflows/publish.yml + with: + runsOn: ubuntu-22.04 + images: ${{ needs.build.outputs.images }} + registry: ghcr.io + secrets: inherit + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..2a39e87 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,72 @@ +--- +name: Publish images to Docker container registries + +env: + # https://github.com/docker/metadata-action?tab=readme-ov-file#environment-variables + DOCKER_METADATA_PR_HEAD_SHA: true + REPO_NAME: ${{ github.repository.split('/')[1] }} + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + registry: + description: Docker container registry + required: true + type: string + +jobs: + + release: + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 30 + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Read build variables + id: build_vars + run: | + vars=$(cat build.json | jq -c '[.variable | to_entries[] | {"key": .key, "value": .value.default}] | from_entries') + echo "vars=$vars" | tee -a "${GITHUB_OUTPUT}" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + env: ${{ fromJSON(steps.build_vars.outputs.vars) }} + with: + # e.g. ghcr.io/cnts4sci/python + images: ${{ inputs.registry }}/${{ github.repository_owner }}/${{ env.REPO_NAME }} + tags: | + type=edge,enable={{is_default_branch}} + type=match,pattern=(\d{4}\.\d{4}(-.+)?),group=1 + type=raw,value=v${{ env.VERSION }},enable=${{ github.ref_type == 'tag' && startsWith(github.ref_name, 'v') }} + + - name: Determine source image + id: images + run: | + src=$(echo '${{ inputs.images }}'| jq -cr '.[("${{ env.REPO_NAME }}"|ascii_upcase|sub("-"; "_"; "g")) + "_IMAGE"]') + echo "src=$src" | tee -a "${GITHUB_OUTPUT}" + + - name: Push image + uses: akhilerm/tag-push-action@v2.2.0 + with: + src: ${{ steps.images.outputs.src }} + dst: ${{ steps.meta.outputs.tags }} + diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4dcb0dd --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,55 @@ +--- +name: Test newly built images + +on: + workflow_call: + inputs: + runsOn: + description: GitHub Actions Runner image + required: true + type: string + images: + description: Images built in build step + required: true + type: string + integration: + description: Run integration tests + required: false + type: boolean + +jobs: + + test: + name: ${{ inputs.integration && inputs.runsOn }} + runs-on: ${{ inputs.runsOn }} + timeout-minutes: 20 + + steps: + + - name: Checkout Repo ⚡️ + uses: actions/checkout@v4 + + - name: Login to GitHub Container Registry 🔑 + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run container checking libraries exist in the container + run: | + docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which python" >> /tmp/cat-output.txt + docker run --rm ${{ needs.build.outputs.image }} /bin/bash -c "which pip3" >> /tmp/cat-output.txt + if cat /tmp/cat-output.txt | grep -q "/opt/python/bin/python"; then + echo "python found" + else + echo "python not found" + exit 1 + fi + + if cat /tmp/cat-output.txt | grep -q "/opt/python/bin/pip3"; then + echo "pip3 found" + else + echo "pip3 not found" + exit 1 + fi diff --git a/Dockerfile b/Dockerfile index 49fc8ba..c24b82f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,35 +7,13 @@ FROM build-base-image AS python-builder # Install dependencies for building Python ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y \ -# libncurses5-dev \ -# libncursesw5-dev \ -# libreadline-dev \ -# libsqlite3-dev \ -# libgdbm-dev \ -# libdb5.3-dev \ -# libbz2-dev \ -# libexpat1-dev \ -# liblzma-dev \ -# tk-dev \ -# libncursesw5-dev \ -# libxml2-dev \ -# libxmlsec1-dev \ -# liblzma-dev \ -# build-essential \ -# wget \ -# curl \ -# llvm \ -# xz-utils \ -# tk-dev \ libffi-dev \ libssl-dev \ zlib1g-dev \ - cmake \ - make && \ + # cmake can be removed after bm:2024:1002 + cmake && \ apt-get clean && rm -rf /var/lib/apt/lists/* - - # Download and extract Python source RUN cd / && git clone https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git @@ -58,6 +36,6 @@ FROM runtime-base-image COPY --from=python-builder /opt/python /opt/python -# Test install aiida-core -RUN /opt/python/bin/python -m pip install aiida-core - +ENV PATH=/opt/python/bin:${PATH} +ENV LD_LIBRARY_PATH=/opt/python/lib:${LD_LIBRARY_PATH} +ENV C_INCLUDE_PATH=/opt/python/include:${C_INCLUDE_PATH} diff --git a/README.md b/README.md index bcf4db9..408a54e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,23 @@ -# Quantum ESPRESSO 7.0 compiled with MPICH 3.1.4 +# Image with purely build Python -The Quantum ESPRESSO v7.0 (only pw.x supported at the moment) is compiled with MPICH 3.1.4 which support ABI interface. -Based on [`container4hpc/base-mpich314:0.1.0`](https://hub.docker.com/repository/docker/container4hpc/base-mpich314) +The image has Python build from cmake into `/opt/python`, in order to have a portable python folder that can be easily moved to another image. +## How to use in multi-stage image build + +To use the python built from this image ported to another image, do: + +``` +FROM ghcr.io/cnts4sci/python: as python-carrier + +FROM + +COPY --from=python-carrier /opt/python /opt/python + +# You may want to set PYTHON_PATH for your system user of the image +USER + +# set up your python path +ENV PATH=/opt/python/bin:${PATH} +ENV LD_LIBRARY_PATH=/opt/python/lib:${LD_LIBRARY_PATH} +ENV C_INCLUDE_PATH=/opt/python/include:${C_INCLUDE_PATH} +``` diff --git a/build.json b/build.json index 5db8230..edef572 100644 --- a/build.json +++ b/build.json @@ -1,11 +1,5 @@ { "variable": { - "BUILD_BASE_IMAGE": { - "default": "ghcr.io/cnts4sci/bm:edge" - }, - "RUNTIME_BASE_IMAGE": { - "default": "docker.io/phusion/baseimage:focal-1.2.0" - }, "PYTHON_VERSION": { "default": "3.9.17" } diff --git a/docker-bake.hcl b/docker-bake.hcl index 76771c1..bb545cc 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -6,12 +6,6 @@ variable "ORGANIZATION" { default = "cnts4sci" } -variable "BULID_BASE_IMAGE" { -} - -variable "RUNTIME_BASE_IMAGE" { -} - variable "PYTHON_VERSION" { } @@ -34,8 +28,8 @@ target "python" { inherits = ["python-meta"] context = "." contexts = { - build-base-image = "docker-image://${BUILD_BASE_IMAGE}" - runtime-base-image = "docker-image://${RUNTIME_BASE_IMAGE}" + build-base-image = "docker-image://ghcr.io/cnts4sci/bm:2024.1001" + runtime-base-image = "docker-image://ghcr.io/cnts4sci/bm:2024.1001" } args = { "PYTHON_VERSION" = "${PYTHON_VERSION}"