From 0fd1bb26d5d39b6b632697993641316799f9a088 Mon Sep 17 00:00:00 2001 From: darthmaim Date: Thu, 19 Sep 2024 11:44:59 +0200 Subject: [PATCH] Use docker artifacts instead of pushing intermediate images to registry --- .github/actions/pull-images/action.yml | 40 +++++++++++++-------- .github/workflows/ci.yml | 10 +++--- .github/workflows/docker-build-and-test.yml | 25 ++++++++----- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/.github/actions/pull-images/action.yml b/.github/actions/pull-images/action.yml index 940fd31bd..49f4147ba 100644 --- a/.github/actions/pull-images/action.yml +++ b/.github/actions/pull-images/action.yml @@ -4,27 +4,37 @@ description: Composite task to pull docker images inputs: platform: description: The platform (amd64/arm64) - required: true + required: false + default: '*' + target: + description: The target image to use (web/...) + required: false + default: '*' + +outputs: + digests: + description: Digests of the loaded images + value: ${{ steps.digests.outputs.digests }} runs: using: composite steps: - - name: Download digests + - name: Download images uses: actions/download-artifact@v4 with: - path: /tmp/digests - pattern: docker-digests-${{ inputs.platform }}-* + path: /tmp/images + pattern: docker-image-${{ inputs.target }}-${{ inputs.platform }} merge-multiple: true - - name: Pull docker images + - name: Load images + shell: bash + run: ls /tmp/images/image-*.tar | xargs -L1 docker image load --input + - name: Docker images + shell: bash + run: docker images --digests + - name: Output digests + id: digests shell: bash run: | - DOCKER_PREFIX=ghcr.io/${{ github.repository }} - DOCKER_PREFIX=$(echo $DOCKER_PREFIX | tr '[A-Z]' '[a-z]') - for TARGET in web worker legacy-importer database-migration e2e - do - digests=(/tmp/digests/"$TARGET"/*) - digest=$(basename $digests) - echo "Pulling $TARGET (sha256:$digest) image" - docker pull "$DOCKER_PREFIX/$TARGET@sha256:${digest}" - docker tag "$DOCKER_PREFIX/$TARGET@sha256:${digest}" gw2treasures/$TARGET - done + DIGESTS=$(echo $(cat /tmp/images/image-*.sha256)) + echo "digests=$DIGESTS" + echo "digests=$DIGESTS" >> $GITHUB_OUTPUT diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3875018b..cbefb1ccf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -145,7 +145,7 @@ jobs: publish-docker: name: Publish / Docker / ${{ matrix.target }} runs-on: ubuntu-latest - if: github.event_name != 'merge_group' + if: github.event_name != 'merge_group' && !github.event.pull_request.head.repo.fork needs: [success] permissions: id-token: write @@ -165,11 +165,11 @@ jobs: IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') echo "IMAGE_ID=$IMAGE_ID" >> "$GITHUB_ENV" - - name: Download digests + - name: Download image digests uses: actions/download-artifact@v4 with: - path: /tmp/digests - pattern: docker-digests-* + path: /tmp/images + pattern: docker-digest-${{ matrix.target }}-* merge-multiple: true - name: Docker meta @@ -200,7 +200,7 @@ jobs: id: image run: | ARGS=$(jq -cr '(.tags | map("--tag " + (. | @sh))) + (.annotations | map("--annotation " + (. | @sh))) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") - SOURCES=$(find /tmp/digests/${{ matrix.target }} -type f -printf '${{ env.IMAGE_ID }}@sha256:%f ') + SOURCES=$(printf "${{ env.IMAGE_ID }}@%s" $(cat /tmp/images/image-*.sha256)) echo "$ARGS" echo "$SOURCES" diff --git a/.github/workflows/docker-build-and-test.yml b/.github/workflows/docker-build-and-test.yml index 7f2247fe0..2fd0032d2 100644 --- a/.github/workflows/docker-build-and-test.yml +++ b/.github/workflows/docker-build-and-test.yml @@ -37,7 +37,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_ID }} + images: gw2treasures/${{ matrix.target }} bake-target: ${{ matrix.target }} - name: Build the Docker image id: build @@ -50,21 +50,28 @@ jobs: ${{ steps.meta.outputs.bake-file-labels }} targets: ${{ matrix.target }} set: | - ${{ matrix.target }}.output=type=image,push-by-digest=true,name-canonical=true,push=true + ${{ !github.event.pull_request.head.repo.fork && format('{0}.output=type=image,push-by-digest=true,name-canonical=true,push=true', matrix.target) }} + ${{ matrix.target }}.output=type=docker,dest=/tmp/image-${{ matrix.target }}-${{ inputs.platform }}.tar,name=gw2treasures/${{ matrix.target }} ${{ matrix.target }}.tags=${{ env.IMAGE_ID }} *.platform=linux/${{ inputs.platform }} *.cache-from=type=gha,scope=build-${{ inputs.platform }}-${{ matrix.target }} *.cache-to=type=gha,scope=build-${{ inputs.platform }}-${{ matrix.target }},mode=max - name: Export digest run: | - mkdir -p /tmp/digests/${{ matrix.target }} digest=$(jq -cr '."${{ matrix.target }}"."containerimage.digest"' <<< '${{ steps.build.outputs.metadata }}') - touch "/tmp/digests/${{ matrix.target }}/${digest#sha256:}" - - name: Upload digest + echo "${digest}" > "/tmp/image-${{ matrix.target }}-${{ inputs.platform }}.sha256" + - name: Upload images uses: actions/upload-artifact@v4 with: - name: docker-digests-${{ inputs.platform }}-${{ matrix.target }} - path: /tmp/digests/* + name: docker-image-${{ matrix.target }}-${{ inputs.platform }} + path: /tmp/image-${{ matrix.target }}-${{ inputs.platform }}.* + if-no-files-found: error + retention-days: 1 + - name: Upload digests + uses: actions/upload-artifact@v4 + with: + name: docker-digest-${{ matrix.target }}-${{ inputs.platform }} + path: /tmp/image-${{ matrix.target }}-${{ inputs.platform }}.sha256 if-no-files-found: error retention-days: 1 @@ -81,7 +88,7 @@ jobs: with: platform: ${{ inputs.platform }} - name: Start docker compose - run: docker compose up -d + run: docker compose up -d --no-build - name: Wait 30s run: sleep 30 - run: docker compose ps -a @@ -128,7 +135,7 @@ jobs: with: platform: ${{ inputs.platform }} - name: Start docker compose - run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d web database database-migration + run: docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d --no-build web database database-migration - name: Wait for containers to be up run: sleep 30 - name: Run e2e tests