From 2b5a823f5c628ff2f395fe1f87a3147cea9d9a42 Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 10:56:51 +0100 Subject: [PATCH 1/6] create base docker image --- src/docker/base/Dockerfile | 45 ++++++++++++++++++++++++++++++++ src/docker/server/Dockerfile | 50 +++--------------------------------- 2 files changed, 48 insertions(+), 47 deletions(-) create mode 100644 src/docker/base/Dockerfile diff --git a/src/docker/base/Dockerfile b/src/docker/base/Dockerfile new file mode 100644 index 000000000000..9cd410ba3fb5 --- /dev/null +++ b/src/docker/base/Dockerfile @@ -0,0 +1,45 @@ +# Copyright 2023 Flower Labs GmbH. All Rights Reserved. + +ARG UBUNTU_VERSION=22.04 +FROM ubuntu:$UBUNTU_VERSION as base + +ENV DEBIAN_FRONTEND noninteractive +# Send stdout and stderr stream directly to the terminal. Ensures that no +# output is retained in a buffer if the application crashes. +ENV PYTHONUNBUFFERED 1 +# Typically, bytecode is created on the first invocation to speed up following invocation. +# However, in Docker we only make a single invocation (when we start the container). +# Therefore, we can disable bytecode writing. +ENV PYTHONDONTWRITEBYTECODE 1 +# Ensure that python encoding is always UTF-8. +ENV PYTHONIOENCODING UTF-8 +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 + +# Install system dependencies +RUN apt-get update \ + && apt-get -y --no-install-recommends install \ + clang-format git unzip ca-certificates openssh-client liblzma-dev \ + build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev wget\ + libsqlite3-dev curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev \ + libxmlsec1-dev libffi-dev liblzma-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install PyEnv and Python +ARG PYTHON_VERSION +ENV PYENV_ROOT /root/.pyenv +ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH +# https://github.com/hadolint/hadolint/wiki/DL4006 +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash +RUN pyenv install ${PYTHON_VERSION} \ + && pyenv global ${PYTHON_VERSION} \ + && pyenv rehash + +# Install specific version of pip +ARG PIP_VERSION +RUN python -m pip install --no-cache-dir pip==$PIP_VERSION + +# Install specific version of setuptools +ARG SETUPTOOLS_VERSION +RUN python -m pip install --no-cache-dir setuptools==$SETUPTOOLS_VERSION diff --git a/src/docker/server/Dockerfile b/src/docker/server/Dockerfile index 37f1ea91f33c..2286e976ce6b 100644 --- a/src/docker/server/Dockerfile +++ b/src/docker/server/Dockerfile @@ -1,51 +1,7 @@ # Copyright 2023 Flower Labs GmbH. All Rights Reserved. -ARG UBUNTU_VERSION=22.04 -FROM ubuntu:$UBUNTU_VERSION as base - -ENV DEBIAN_FRONTEND noninteractive -# Send stdout and stderr stream directly to the terminal. Ensures that no -# output is retained in a buffer if the application crashes. -ENV PYTHONUNBUFFERED 1 -# Typically, bytecode is created on the first invocation to speed up following invocation. -# However, in Docker we only make a single invocation (when we start the container). -# Therefore, we can disable bytecode writing. -ENV PYTHONDONTWRITEBYTECODE 1 -# Ensure that python encoding is always UTF-8. -ENV PYTHONIOENCODING UTF-8 -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 - -# Install system dependencies -RUN apt-get update \ - && apt-get -y --no-install-recommends install \ - clang-format git unzip ca-certificates openssh-client liblzma-dev \ - build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev wget\ - libsqlite3-dev curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev \ - libxmlsec1-dev libffi-dev liblzma-dev \ - && rm -rf /var/lib/apt/lists/* - -# Install PyEnv and Python -ARG PYTHON_VERSION -ENV PYENV_ROOT /root/.pyenv -ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH -# https://github.com/hadolint/hadolint/wiki/DL4006 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash -RUN pyenv install ${PYTHON_VERSION} \ - && pyenv global ${PYTHON_VERSION} \ - && pyenv rehash - -# Install specific version of pip -ARG PIP_VERSION -RUN python -m pip install --no-cache-dir pip==$PIP_VERSION - -# Install specific version of setuptools -ARG SETUPTOOLS_VERSION -RUN python -m pip install --no-cache-dir setuptools==$SETUPTOOLS_VERSION - -# Server image -FROM base as server +ARG BASE_VERSION=22.04 +FROM flwr/base:$BASE_VERSION WORKDIR /app ARG FLWR_VERSION @@ -53,5 +9,5 @@ RUN python -m pip install -U --no-cache-dir flwr[rest]==${FLWR_VERSION} ENTRYPOINT ["python", "-c", "from flwr.server import run_server; run_server()"] # Test if Flower can be successfully installed and imported -FROM server as test +FROM flwr/base:$BASE_VERSION RUN python -c "from flwr.server import run_server" From 03b90a6d44367598199fe28ab36fddcd3e5bbe57 Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 11:37:40 +0100 Subject: [PATCH 2/6] make reusable workflow more general --- .../{_docker-server.yml => _docker-build.yml} | 100 ++++-------------- .github/workflows/docker-server.yml | 54 ++++++++-- 2 files changed, 66 insertions(+), 88 deletions(-) rename .github/workflows/{_docker-server.yml => _docker-build.yml} (50%) diff --git a/.github/workflows/_docker-server.yml b/.github/workflows/_docker-build.yml similarity index 50% rename from .github/workflows/_docker-server.yml rename to .github/workflows/_docker-build.yml index 2e8372e1ede5..88851db8335a 100644 --- a/.github/workflows/_docker-server.yml +++ b/.github/workflows/_docker-build.yml @@ -3,25 +3,21 @@ name: Reusable docker server image build workflow on: workflow_call: inputs: - flwr-version: - description: "Version of Flower." + namespace_repository: + description: "The namespace and repository in the following format `namespace/repository` e.g. (flwr/base)." required: true type: string - python-version: - description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to 3.11." - required: false - type: string - pip-version: - description: "Version of pip. Defaults to the version defined in actions/bootstrap." - required: false + file: + description: "Path to the Dockerfile." + required: true type: string - setuptools-version: - description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." + build-args: + description: "List of build-time variables." required: false type: string - ubuntu-version: - description: "Version of Ubuntu. Defaults to 22.04." - required: false + tags: + description: "List of tags." + required: true type: string secrets: dockerhub-user: @@ -33,64 +29,15 @@ on: description: "Metadata of the docker image." value: ${{ jobs.build-manifest.outputs.metadata }} -env: - REGISTRY_IMAGE: flwr/server - DEFAULT_PYTHON: 3.11 - DEFAULT_UBUNTU: 22.04 - permissions: contents: read # based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners jobs: - parameters: - name: Collect build parameters - runs-on: ubuntu-22.04 - timeout-minutes: 10 - outputs: - pip-version: ${{ steps.versions.outputs.pip-version }} - setuptools-version: ${{ steps.versions.outputs.setuptools-version }} - python-version: ${{ steps.versions.outputs.python-version }} - ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }} - - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - uses: ./.github/actions/bootstrap - if: ${{ !(inputs.pip-version != '' && inputs.setuptools-version != '') }} - id: bootstrap - - - id: versions - run: | - if [[ "${{ inputs.pip-version }}" = "" ]]; then - echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" - else - echo "pip-version=${{ inputs.pip-version }}" >> "$GITHUB_OUTPUT" - fi - - if [[ "${{ inputs.setuptools-version }}" = "" ]]; then - echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - else - echo "setuptools-version=${{ inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - fi - - if [[ "${{ inputs.python-version }}" = "" ]]; then - echo "python-version=${{ env.DEFAULT_PYTHON }}" >> "$GITHUB_OUTPUT" - else - echo "python-version=${{ inputs.python-version }}" >> "$GITHUB_OUTPUT" - fi - - if [[ "${{ inputs.ubuntu-version }}" = "" ]]; then - echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT" - else - echo "ubuntu-version=${{ inputs.ubuntu-version }}" >> "$GITHUB_OUTPUT" - fi - build: name: Build server image runs-on: ubuntu-22.04 timeout-minutes: 60 - needs: parameters strategy: fail-fast: true matrix: @@ -98,7 +45,7 @@ jobs: # build-push action and qemu use different platform names # therefore we create a map { qemu: "", docker: "linux/amd64" }, - { qemu: "arm64", docker: "linux/arm64" }, + # { qemu: "arm64", docker: "linux/arm64" }, ] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -113,7 +60,7 @@ jobs: id: meta uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 with: - images: ${{ env.REGISTRY_IMAGE }} + images: ${{ inputs.namespace_repository }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 @@ -129,14 +76,9 @@ jobs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: platforms: ${{ matrix.platform.docker }} - context: src/docker/server - build-args: | - PYTHON_VERSION=${{ needs.parameters.outputs.python-version }} - PIP_VERSION=${{ needs.parameters.outputs.pip-version }} - SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} - FLWR_VERSION=${{ inputs.flwr-version }} - UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }} - outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + context: ${{ inputs.file }} + build-args: ${{ inputs.build-args }} + outputs: type=image,name=${{ inputs.namespace_repository }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -156,7 +98,7 @@ jobs: name: Build and push docker manifest for all platforms runs-on: ubuntu-22.04 timeout-minutes: 10 - needs: [parameters, build] + needs: build outputs: metadata: ${{ steps.meta.outputs.json }} steps: @@ -170,10 +112,8 @@ jobs: id: meta uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 with: - images: ${{ env.REGISTRY_IMAGE }} - tags: | - ${{ inputs.flwr-version }}-py${{ needs.parameters.outputs.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }} - ${{ inputs.flwr-version }} + images: ${{ inputs.namespace_repository }} + tags: ${{ inputs.tags }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 @@ -188,6 +128,6 @@ jobs: working-directory: /tmp/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) + $(printf '${{ inputs.namespace_repository }}@sha256:%s ' *) - name: Inspect image - run: docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} + run: docker buildx imagetools inspect ${{ inputs.namespace_repository }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml index 6f57c6a0691b..7802af6b5079 100644 --- a/.github/workflows/docker-server.yml +++ b/.github/workflows/docker-server.yml @@ -8,34 +8,72 @@ on: required: true type: string python-version: - description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to the version defined in _docker-server.yaml." + description: "Version of Python e.g. (3.11.7 or 3.11)" required: false type: string + default: "3.11" pip-version: - description: "Version of pip. Defaults to the version defined in _docker-server.yaml." + description: "Version of pip. Defaults to the version defined in actions/bootstrap." required: false type: string setuptools-version: - description: "Version of setuptools. Defaults to the version defined in _docker-server.yaml." + description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." required: false type: string ubuntu-version: description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml." required: false type: string + default: "22.04" permissions: contents: read jobs: + parameters: + name: Collect build parameters + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + pip-version: ${{ steps.versions.outputs.pip-version }} + setuptools-version: ${{ steps.versions.outputs.setuptools-version }} + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: ./.github/actions/bootstrap + if: ${{ !(github.event.inputs.pip-version != '' && github.event.inputs.setuptools-version != '') }} + id: bootstrap + + - id: versions + run: | + if [[ "${{ github.event.inputs.pip-version }}" = "" ]]; then + echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" + else + echo "pip-version=${{ github.event.inputs.pip-version }}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ github.event.inputs.setuptools-version }}" = "" ]]; then + echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + else + echo "setuptools-version=${{ github.event.inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + fi + build-server-images: uses: ./.github/workflows/_docker-server.yml + needs: parameters with: - flwr-version: ${{ github.event.inputs.flwr-version }} - python-version: ${{ github.event.inputs.python-version }} - pip-version: ${{ github.event.inputs.pip-version }} - setuptools-version: ${{ github.event.inputs.setuptools-version }} - ubuntu-version: ${{ github.event.inputs.ubuntu-version }} + namespace_repository: flwr/server + file: src/docker/server + build-args: | + PYTHON_VERSION=${{ github.event.inputs.python-version }} + PIP_VERSION=${{ needs.parameters.outputs.pip-version }} + SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} + FLWR_VERSION=${{ github.event.inputs.flwr-version }} + UBUNTU_VERSION=${{ github.event.inputs.ubuntu-version }} + tags: | + ${{ github.event.inputs.flwr-version }}-py${{ github.event.inputs.python-version }}-ubuntu${{ github.event.inputs.ubuntu-version }} + ${{ github.event.inputs.flwr-version }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} From 860451292d6604b03bebb40d51cb887b0da7dcf4 Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 11:40:33 +0100 Subject: [PATCH 3/6] build base image ci --- .github/workflows/docker-base.yml | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/docker-base.yml diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml new file mode 100644 index 000000000000..2ab12a00163e --- /dev/null +++ b/.github/workflows/docker-base.yml @@ -0,0 +1,85 @@ +name: Build docker base image + +on: + workflow_dispatch: + inputs: + python-version: + description: "Version of Python e.g. (3.11.7 or 3.11)" + required: false + type: string + default: "3.11" + pip-version: + description: "Version of pip. Defaults to the version defined in actions/bootstrap." + required: false + type: string + setuptools-version: + description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." + required: false + type: string + ubuntu-version: + description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml." + required: false + type: string + default: "22.04" + +permissions: + contents: read + +jobs: + parameters: + name: Collect build parameters + runs-on: ubuntu-22.04 + timeout-minutes: 10 + outputs: + pip-version: ${{ steps.versions.outputs.pip-version }} + setuptools-version: ${{ steps.versions.outputs.setuptools-version }} + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: ./.github/actions/bootstrap + if: ${{ !(github.event.inputs.pip-version != '' && github.event.inputs.setuptools-version != '') }} + id: bootstrap + + - id: versions + run: | + if [[ "${{ github.event.inputs.pip-version }}" = "" ]]; then + echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" + else + echo "pip-version=${{ github.event.inputs.pip-version }}" >> "$GITHUB_OUTPUT" + fi + + if [[ "${{ github.event.inputs.setuptools-version }}" = "" ]]; then + echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + else + echo "setuptools-version=${{ github.event.inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + fi + + build-server-images: + uses: ./.github/workflows/_docker-server.yml + needs: parameters + with: + namespace_repository: flwr/base + file: src/docker/server + build-args: | + PYTHON_VERSION=${{ github.event.inputs.python-version }} + PIP_VERSION=${{ needs.parameters.outputs.pip-version }} + SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} + UBUNTU_VERSION=${{ github.event.inputs.ubuntu-version }} + tags: | + py${{ github.event.inputs.python-version }}-ubuntu${{ github.event.inputs.ubuntu-version }} + test + secrets: + dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} + + summary: + runs-on: ubuntu-22.04 + needs: build-server-images + steps: + - run: | + echo "### Images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + for IMAGE in $(echo ${{ toJson(needs.build-server-images.outputs.metadata) }} | jq -r '.tags[]' ); do + echo "- $IMAGE" >> $GITHUB_STEP_SUMMARY + done From 1fb648cd17f3654b040f711e94fc0bb3fb9e2927 Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 11:41:51 +0100 Subject: [PATCH 4/6] create base images ci --- .github/workflows/_docker-build.yml | 6 +-- .github/workflows/docker-base.yml | 68 ++++++++--------------------- .github/workflows/docker-server.yml | 57 +++--------------------- src/docker/server/Dockerfile | 4 +- 4 files changed, 28 insertions(+), 107 deletions(-) diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml index 88851db8335a..817ce348705d 100644 --- a/.github/workflows/_docker-build.yml +++ b/.github/workflows/_docker-build.yml @@ -45,11 +45,9 @@ jobs: # build-push action and qemu use different platform names # therefore we create a map { qemu: "", docker: "linux/amd64" }, - # { qemu: "arm64", docker: "linux/arm64" }, + { qemu: "arm64", docker: "linux/arm64" }, ] steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Set up QEMU if: matrix.platform.qemu != '' uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 @@ -76,7 +74,7 @@ jobs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: platforms: ${{ matrix.platform.docker }} - context: ${{ inputs.file }} + context: "{{defaultContext}}:${{ inputs.file }}" build-args: ${{ inputs.build-args }} outputs: type=image,name=${{ inputs.namespace_repository }},push-by-digest=true,name-canonical=true,push=true diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml index 2ab12a00163e..bddc70385212 100644 --- a/.github/workflows/docker-base.yml +++ b/.github/workflows/docker-base.yml @@ -1,30 +1,16 @@ name: Build docker base image on: - workflow_dispatch: - inputs: - python-version: - description: "Version of Python e.g. (3.11.7 or 3.11)" - required: false - type: string - default: "3.11" - pip-version: - description: "Version of pip. Defaults to the version defined in actions/bootstrap." - required: false - type: string - setuptools-version: - description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." - required: false - type: string - ubuntu-version: - description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml." - required: false - type: string - default: "22.04" + push: + # paths: + # - src/docker/base permissions: contents: read +env: + DEFAULT_UBUNTU: 22.04 + jobs: parameters: name: Collect build parameters @@ -33,53 +19,35 @@ jobs: outputs: pip-version: ${{ steps.versions.outputs.pip-version }} setuptools-version: ${{ steps.versions.outputs.setuptools-version }} + ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }} steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - uses: ./.github/actions/bootstrap - if: ${{ !(github.event.inputs.pip-version != '' && github.event.inputs.setuptools-version != '') }} id: bootstrap - id: versions run: | - if [[ "${{ github.event.inputs.pip-version }}" = "" ]]; then - echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" - else - echo "pip-version=${{ github.event.inputs.pip-version }}" >> "$GITHUB_OUTPUT" - fi - - if [[ "${{ github.event.inputs.setuptools-version }}" = "" ]]; then - echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - else - echo "setuptools-version=${{ github.event.inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - fi + echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" + echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" + echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT" build-server-images: - uses: ./.github/workflows/_docker-server.yml + uses: ./.github/workflows/_docker-build.yml needs: parameters + strategy: + matrix: + python-version: [3.11] with: namespace_repository: flwr/base - file: src/docker/server + file: src/docker/base build-args: | - PYTHON_VERSION=${{ github.event.inputs.python-version }} + PYTHON_VERSION=${{ matrix.python-version }} PIP_VERSION=${{ needs.parameters.outputs.pip-version }} SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} - UBUNTU_VERSION=${{ github.event.inputs.ubuntu-version }} - tags: | - py${{ github.event.inputs.python-version }}-ubuntu${{ github.event.inputs.ubuntu-version }} - test + UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }} + tags: py${{ matrix.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} - - summary: - runs-on: ubuntu-22.04 - needs: build-server-images - steps: - - run: | - echo "### Images" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - for IMAGE in $(echo ${{ toJson(needs.build-server-images.outputs.metadata) }} | jq -r '.tags[]' ); do - echo "- $IMAGE" >> $GITHUB_STEP_SUMMARY - done diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml index 7802af6b5079..3e351c6e9b43 100644 --- a/.github/workflows/docker-server.yml +++ b/.github/workflows/docker-server.yml @@ -7,21 +7,8 @@ on: description: "Version of Flower e.g. (1.6.0)." required: true type: string - python-version: - description: "Version of Python e.g. (3.11.7 or 3.11)" - required: false - type: string - default: "3.11" - pip-version: - description: "Version of pip. Defaults to the version defined in actions/bootstrap." - required: false - type: string - setuptools-version: - description: "Version of setuptools. Defaults to the version defined in actions/bootstrap." - required: false - type: string - ubuntu-version: - description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml." + base-image-version: + description: "Version of Flower base image." required: false type: string default: "22.04" @@ -30,50 +17,18 @@ permissions: contents: read jobs: - parameters: - name: Collect build parameters - runs-on: ubuntu-22.04 - timeout-minutes: 10 - outputs: - pip-version: ${{ steps.versions.outputs.pip-version }} - setuptools-version: ${{ steps.versions.outputs.setuptools-version }} - - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - uses: ./.github/actions/bootstrap - if: ${{ !(github.event.inputs.pip-version != '' && github.event.inputs.setuptools-version != '') }} - id: bootstrap - - - id: versions - run: | - if [[ "${{ github.event.inputs.pip-version }}" = "" ]]; then - echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" - else - echo "pip-version=${{ github.event.inputs.pip-version }}" >> "$GITHUB_OUTPUT" - fi - - if [[ "${{ github.event.inputs.setuptools-version }}" = "" ]]; then - echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - else - echo "setuptools-version=${{ github.event.inputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - fi - build-server-images: - uses: ./.github/workflows/_docker-server.yml - needs: parameters + uses: ./.github/workflows/_docker-build.yml with: namespace_repository: flwr/server file: src/docker/server build-args: | - PYTHON_VERSION=${{ github.event.inputs.python-version }} - PIP_VERSION=${{ needs.parameters.outputs.pip-version }} - SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} FLWR_VERSION=${{ github.event.inputs.flwr-version }} - UBUNTU_VERSION=${{ github.event.inputs.ubuntu-version }} + BASE_IMAGE_VERSION=${{ github.event.inputs.base-image-version }} tags: | - ${{ github.event.inputs.flwr-version }}-py${{ github.event.inputs.python-version }}-ubuntu${{ github.event.inputs.ubuntu-version }} + ${{ github.event.inputs.flwr-version }}-${{ github.event.inputs.base-image-version }} ${{ github.event.inputs.flwr-version }} + latest secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/src/docker/server/Dockerfile b/src/docker/server/Dockerfile index 2286e976ce6b..a20864e7d7bf 100644 --- a/src/docker/server/Dockerfile +++ b/src/docker/server/Dockerfile @@ -1,7 +1,7 @@ # Copyright 2023 Flower Labs GmbH. All Rights Reserved. -ARG BASE_VERSION=22.04 -FROM flwr/base:$BASE_VERSION +ARG BASE_IMAGE_VERSION=22.04 +FROM flwr/base:$BASE_IMAGE_VERSION WORKDIR /app ARG FLWR_VERSION From 684ee32d26f547e60efc0cf85d7b158d4787e5ef Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 15:21:36 +0100 Subject: [PATCH 5/6] update server image build workflow --- .github/workflows/_docker-build.yml | 18 +++++++++--------- .github/workflows/docker-base.yml | 12 +++++++----- .github/workflows/docker-server.yml | 11 +++++++---- src/docker/server/Dockerfile | 6 +++--- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml index 817ce348705d..d36076444746 100644 --- a/.github/workflows/_docker-build.yml +++ b/.github/workflows/_docker-build.yml @@ -3,12 +3,12 @@ name: Reusable docker server image build workflow on: workflow_call: inputs: - namespace_repository: + namespace-repository: description: "The namespace and repository in the following format `namespace/repository` e.g. (flwr/base)." required: true type: string - file: - description: "Path to the Dockerfile." + file-dir: + description: "Path of the directory that contains the Dockerfile." required: true type: string build-args: @@ -58,7 +58,7 @@ jobs: id: meta uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 with: - images: ${{ inputs.namespace_repository }} + images: ${{ inputs.namespace-repository }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0 @@ -74,9 +74,9 @@ jobs: uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: platforms: ${{ matrix.platform.docker }} - context: "{{defaultContext}}:${{ inputs.file }}" + context: "{{defaultContext}}:${{ inputs.file-dir }}" build-args: ${{ inputs.build-args }} - outputs: type=image,name=${{ inputs.namespace_repository }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | @@ -110,7 +110,7 @@ jobs: id: meta uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0 with: - images: ${{ inputs.namespace_repository }} + images: ${{ inputs.namespace-repository }} tags: ${{ inputs.tags }} - name: Set up Docker Buildx @@ -126,6 +126,6 @@ jobs: working-directory: /tmp/digests run: | docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ inputs.namespace_repository }}@sha256:%s ' *) + $(printf '${{ inputs.namespace-repository }}@sha256:%s ' *) - name: Inspect image - run: docker buildx imagetools inspect ${{ inputs.namespace_repository }}:${{ steps.meta.outputs.version }} + run: docker buildx imagetools inspect ${{ inputs.namespace-repository }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml index bddc70385212..a3895632d184 100644 --- a/.github/workflows/docker-base.yml +++ b/.github/workflows/docker-base.yml @@ -2,8 +2,8 @@ name: Build docker base image on: push: - # paths: - # - src/docker/base + branches: ["main"] + paths: ["src/docker/base/**"] permissions: contents: read @@ -33,15 +33,17 @@ jobs: echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT" - build-server-images: + build-base-images: + name: Build images uses: ./.github/workflows/_docker-build.yml needs: parameters strategy: + fail-fast: true matrix: python-version: [3.11] with: - namespace_repository: flwr/base - file: src/docker/base + namespace-repository: flwr/base + file-dir: src/docker/base build-args: | PYTHON_VERSION=${{ matrix.python-version }} PIP_VERSION=${{ needs.parameters.outputs.pip-version }} diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml index 3e351c6e9b43..c710cf588e89 100644 --- a/.github/workflows/docker-server.yml +++ b/.github/workflows/docker-server.yml @@ -8,20 +8,21 @@ on: required: true type: string base-image-version: - description: "Version of Flower base image." + description: "Version of the Flower base image." required: false type: string - default: "22.04" + default: "py3.11-ubuntu22.04" permissions: contents: read jobs: build-server-images: + name: Build images uses: ./.github/workflows/_docker-build.yml with: - namespace_repository: flwr/server - file: src/docker/server + namespace-repository: flwr/server + file-dir: src/docker/server build-args: | FLWR_VERSION=${{ github.event.inputs.flwr-version }} BASE_IMAGE_VERSION=${{ github.event.inputs.base-image-version }} @@ -34,8 +35,10 @@ jobs: dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} summary: + name: Summary runs-on: ubuntu-22.04 needs: build-server-images + timeout-minutes: 5 steps: - run: | echo "### Images" >> $GITHUB_STEP_SUMMARY diff --git a/src/docker/server/Dockerfile b/src/docker/server/Dockerfile index a20864e7d7bf..9bf3214bb42c 100644 --- a/src/docker/server/Dockerfile +++ b/src/docker/server/Dockerfile @@ -1,7 +1,7 @@ # Copyright 2023 Flower Labs GmbH. All Rights Reserved. -ARG BASE_IMAGE_VERSION=22.04 -FROM flwr/base:$BASE_IMAGE_VERSION +ARG BASE_IMAGE_VERSION=py3.11-ubuntu22.04 +FROM flwr/base:$BASE_IMAGE_VERSION as server WORKDIR /app ARG FLWR_VERSION @@ -9,5 +9,5 @@ RUN python -m pip install -U --no-cache-dir flwr[rest]==${FLWR_VERSION} ENTRYPOINT ["python", "-c", "from flwr.server import run_server; run_server()"] # Test if Flower can be successfully installed and imported -FROM flwr/base:$BASE_VERSION +FROM server as test RUN python -c "from flwr.server import run_server" From ce4a953c270cd801dd1081b42cacbd4a07cfcb44 Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Wed, 6 Dec 2023 19:05:13 +0100 Subject: [PATCH 6/6] increase timeout to 10min --- .github/workflows/docker-server.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml index c710cf588e89..325612be2573 100644 --- a/.github/workflows/docker-server.yml +++ b/.github/workflows/docker-server.yml @@ -38,7 +38,7 @@ jobs: name: Summary runs-on: ubuntu-22.04 needs: build-server-images - timeout-minutes: 5 + timeout-minutes: 10 steps: - run: | echo "### Images" >> $GITHUB_STEP_SUMMARY