From 30bc20c898e00ef4ea3de36d4e7b6152d332a73b Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 10:32:54 +0100 Subject: [PATCH] Add CI workflow for building and releasing server docker images (#2682) --- .github/actions/bootstrap/action.yml | 22 ++- .github/workflows/_docker-server.yml | 193 +++++++++++++++++++++++++++ .github/workflows/docker-server.yml | 52 ++++++++ 3 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/_docker-server.yml create mode 100644 .github/workflows/docker-server.yml diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml index 584ae2634d9e..bab5113bc567 100644 --- a/.github/actions/bootstrap/action.yml +++ b/.github/actions/bootstrap/action.yml @@ -4,7 +4,7 @@ inputs: python-version: description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax." default: 3.8 - pip-version: + pip-version: description: "Version of pip to be installed using pip" default: 23.3.1 setuptools-version: @@ -13,6 +13,19 @@ inputs: poetry-version: description: "Version of poetry to be installed using pip" default: 1.5.1 +outputs: + python-version: + description: "Version range or exact version of Python or PyPy" + value: ${{ steps.versions.outputs.python-version }} + pip-version: + description: "Installed version of pip" + value: ${{ steps.versions.outputs.pip-version }} + setuptools-version: + description: "Installed version of setuptools" + value: ${{ steps.versions.outputs.setuptools-version }} + poetry-version: + description: "Installed version of poetry" + value: ${{ steps.versions.outputs.poetry-version }} runs: using: "composite" steps: @@ -27,3 +40,10 @@ runs: python -m pip install -U poetry==${{ inputs.poetry-version }} python -m poetry config virtualenvs.create false shell: bash + - id: versions + shell: bash + run: | + echo "python-version=$(echo ${{ inputs.python-version }})" >> $GITHUB_OUTPUT + echo "pip-version=$(echo ${{ inputs.pip-version }})" >> $GITHUB_OUTPUT + echo "setuptools-version=$(echo ${{ inputs.setuptools-version }})" >> $GITHUB_OUTPUT + echo "poetry-version=$(echo ${{ inputs.poetry-version }})" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_docker-server.yml b/.github/workflows/_docker-server.yml new file mode 100644 index 000000000000..2e8372e1ede5 --- /dev/null +++ b/.github/workflows/_docker-server.yml @@ -0,0 +1,193 @@ +name: Reusable docker server image build workflow + +on: + workflow_call: + inputs: + flwr-version: + description: "Version of Flower." + required: true + type: string + python-version: + description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to 3.11." + required: false + type: string + pip-version: + description: "Version of pip. Defaults to the version defined in actions/bootstrap." + required: false + type: string + setuptools-version: + description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." + required: false + type: string + ubuntu-version: + description: "Version of Ubuntu. Defaults to 22.04." + required: false + type: string + secrets: + dockerhub-user: + required: true + dockerhub-token: + required: true + outputs: + metadata: + description: "Metadata of the docker image." + value: ${{ jobs.build-manifest.outputs.metadata }} + +env: + REGISTRY_IMAGE: flwr/server + DEFAULT_PYTHON: 3.11 + DEFAULT_UBUNTU: 22.04 + +permissions: + contents: read + +# based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners +jobs: + parameters: + name: Collect build parameters + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + pip-version: ${{ steps.versions.outputs.pip-version }} + setuptools-version: ${{ steps.versions.outputs.setuptools-version }} + python-version: ${{ steps.versions.outputs.python-version }} + ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }} + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: ./.github/actions/bootstrap + if: ${{ !(inputs.pip-version != '' && inputs.setuptools-version != '') }} + id: bootstrap + + - id: versions + run: | + if [[ "${{ inputs.pip-version }}" = "" ]]; then + echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" + else + echo "pip-version=${{ inputs.pip-version }}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ inputs.setuptools-version }}" = "" ]]; then + echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + else + echo "setuptools-version=${{ inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ inputs.python-version }}" = "" ]]; then + echo "python-version=${{ env.DEFAULT_PYTHON }}" >> "$GITHUB_OUTPUT" + else + echo "python-version=${{ inputs.python-version }}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ inputs.ubuntu-version }}" = "" ]]; then + echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT" + else + echo "ubuntu-version=${{ inputs.ubuntu-version }}" >> "$GITHUB_OUTPUT" + fi + + build: + name: Build server image + runs-on: ubuntu-22.04 + timeout-minutes: 60 + needs: parameters + strategy: + fail-fast: true + matrix: + platform: [ + # build-push action and qemu use different platform names + # therefore we create a map + { qemu: "", docker: "linux/amd64" }, + { qemu: "arm64", docker: "linux/arm64" }, + ] + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Set up QEMU + if: matrix.platform.qemu != '' + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + with: + platforms: ${{ matrix.platform.qemu }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 + with: + images: ${{ env.REGISTRY_IMAGE }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Login to Docker Hub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.dockerhub-user }} + password: ${{ secrets.dockerhub-token }} + + - name: Build and push + id: build + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 + with: + platforms: ${{ matrix.platform.docker }} + context: src/docker/server + build-args: | + PYTHON_VERSION=${{ needs.parameters.outputs.python-version }} + PIP_VERSION=${{ needs.parameters.outputs.pip-version }} + SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} + FLWR_VERSION=${{ inputs.flwr-version }} + UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: digests + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + build-manifest: + name: Build and push docker manifest for all platforms + runs-on: ubuntu-22.04 + timeout-minutes: 10 + needs: [parameters, build] + outputs: + metadata: ${{ steps.meta.outputs.json }} + steps: + - name: Download digests + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: digests + path: /tmp/digests + + - name: Docker meta + id: meta + uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 + with: + images: ${{ env.REGISTRY_IMAGE }} + tags: | + ${{ inputs.flwr-version }}-py${{ needs.parameters.outputs.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }} + ${{ inputs.flwr-version }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 + + - name: Login to Docker Hub + uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + with: + username: ${{ secrets.dockerhub-user }} + password: ${{ secrets.dockerhub-token }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + - name: Inspect image + run: docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml new file mode 100644 index 000000000000..6f57c6a0691b --- /dev/null +++ b/.github/workflows/docker-server.yml @@ -0,0 +1,52 @@ +name: Build docker server image + +on: + workflow_dispatch: + inputs: + flwr-version: + description: "Version of Flower e.g. (1.6.0)." + required: true + type: string + python-version: + description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to the version defined in _docker-server.yaml." + required: false + type: string + pip-version: + description: "Version of pip. Defaults to the version defined in _docker-server.yaml." + required: false + type: string + setuptools-version: + description: "Version of setuptools. Defaults to the version defined in _docker-server.yaml." + required: false + type: string + ubuntu-version: + description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml." + required: false + type: string + +permissions: + contents: read + +jobs: + build-server-images: + uses: ./.github/workflows/_docker-server.yml + with: + flwr-version: ${{ github.event.inputs.flwr-version }} + python-version: ${{ github.event.inputs.python-version }} + pip-version: ${{ github.event.inputs.pip-version }} + setuptools-version: ${{ github.event.inputs.setuptools-version }} + ubuntu-version: ${{ github.event.inputs.ubuntu-version }} + secrets: + dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + + summary: + runs-on: ubuntu-22.04 + needs: build-server-images + steps: + - run: | + echo "### Images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + for IMAGE in $(echo ${{ toJson(needs.build-server-images.outputs.metadata) }} | jq -r '.tags[]' ); do + echo "- $IMAGE" >> $GITHUB_STEP_SUMMARY + done