Create docker base images (#2688)

adap · Dec 6, 2023 · 0bbe430 · 0bbe430
1 parent 8e9cb56
commit 0bbe430
Show file tree

Hide file tree

Showing 5 changed files with 137 additions and 147 deletions.
diff --git a/.github/workflows/_docker-server.yml → .github/workflows/_docker-build.yml b/.github/workflows/_docker-server.yml → .github/workflows/_docker-build.yml
@@ -3,25 +3,21 @@ name: Reusable docker server image build workflow
 on:
   workflow_call:
     inputs:
-      flwr-version:
-        description: "Version of Flower."
+      namespace-repository:
+        description: "The namespace and repository in the following format `namespace/repository` e.g. (flwr/base)."
         required: true
         type: string
-      python-version:
-        description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to 3.11."
-        required: false
-        type: string
-      pip-version:
-        description: "Version of pip. Defaults to the version defined in actions/bootstrap."
-        required: false
+      file-dir:
+        description: "Path of the directory that contains the Dockerfile."
+        required: true
         type: string
-      setuptools-version:
-        description: "Version of setuptools. Defaults to the version defined in actions/bootstrap."
+      build-args:
+        description: "List of build-time variables."
         required: false
         type: string
-      ubuntu-version:
-        description: "Version of Ubuntu. Defaults to 22.04."
-        required: false
+      tags:
+        description: "List of tags."
+        required: true
         type: string
     secrets:
       dockerhub-user:
@@ -33,64 +29,15 @@ on:
         description: "Metadata of the docker image."
         value: ${{ jobs.build-manifest.outputs.metadata }}
 
-env:
-  REGISTRY_IMAGE: flwr/server
-  DEFAULT_PYTHON: 3.11
-  DEFAULT_UBUNTU: 22.04
-
 permissions:
   contents: read
 
 # based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
 jobs:
-  parameters:
-    name: Collect build parameters
-    runs-on: ubuntu-22.04
-    timeout-minutes: 10
-    outputs:
-      pip-version: ${{ steps.versions.outputs.pip-version }}
-      setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
-      python-version: ${{ steps.versions.outputs.python-version }}
-      ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }}
-
-    steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-      - uses: ./.github/actions/bootstrap
-        if: ${{ !(inputs.pip-version != '' && inputs.setuptools-version != '') }}
-        id: bootstrap
-
-      - id: versions
-        run: |
-          if [[ "${{ inputs.pip-version }}" = "" ]]; then
-              echo "pip-version=${{  steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
-          else
-              echo "pip-version=${{ inputs.pip-version }}" >> "$GITHUB_OUTPUT"
-          fi
-
-          if [[ "${{ inputs.setuptools-version }}" = "" ]]; then
-              echo "setuptools-version=${{  steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
-          else
-              echo "setuptools-version=${{ inputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
-          fi
-
-          if [[ "${{ inputs.python-version }}" = "" ]]; then
-              echo "python-version=${{  env.DEFAULT_PYTHON }}" >> "$GITHUB_OUTPUT"
-          else
-              echo "python-version=${{ inputs.python-version }}" >> "$GITHUB_OUTPUT"
-          fi
-
-          if [[ "${{ inputs.ubuntu-version }}" = "" ]]; then
-              echo "ubuntu-version=${{  env.DEFAULT_UBUNTU  }}" >> "$GITHUB_OUTPUT"
-          else
-              echo "ubuntu-version=${{ inputs.ubuntu-version }}" >> "$GITHUB_OUTPUT"
-          fi
-
   build:
     name: Build server image
     runs-on: ubuntu-22.04
     timeout-minutes: 60
-    needs: parameters
     strategy:
       fail-fast: true
       matrix:
@@ -101,8 +48,6 @@ jobs:
             { qemu: "arm64", docker: "linux/arm64" },
           ]
     steps:
-      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
       - name: Set up QEMU
         if: matrix.platform.qemu != ''
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
@@ -113,7 +58,7 @@ jobs:
         id: meta
         uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0
         with:
-          images: ${{ env.REGISTRY_IMAGE }}
+          images: ${{ inputs.namespace-repository }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
@@ -129,14 +74,9 @@ jobs:
         uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
         with:
           platforms: ${{ matrix.platform.docker }}
-          context: src/docker/server
-          build-args: |
-            PYTHON_VERSION=${{ needs.parameters.outputs.python-version }}
-            PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
-            SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
-            FLWR_VERSION=${{ inputs.flwr-version }}
-            UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }}
-          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+          context: "{{defaultContext}}:${{ inputs.file-dir }}"
+          build-args: ${{ inputs.build-args }}
+          outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
 
       - name: Export digest
         run: |
@@ -156,7 +96,7 @@ jobs:
     name: Build and push docker manifest for all platforms
     runs-on: ubuntu-22.04
     timeout-minutes: 10
-    needs: [parameters, build]
+    needs: build
     outputs:
       metadata: ${{ steps.meta.outputs.json }}
     steps:
@@ -170,10 +110,8 @@ jobs:
         id: meta
         uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0
         with:
-          images: ${{ env.REGISTRY_IMAGE }}
-          tags: |
-            ${{ inputs.flwr-version }}-py${{ needs.parameters.outputs.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }}
-            ${{ inputs.flwr-version }}
+          images: ${{ inputs.namespace-repository }}
+          tags: ${{ inputs.tags }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
@@ -188,6 +126,6 @@ jobs:
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-              $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)
+              $(printf '${{ inputs.namespace-repository }}@sha256:%s ' *)
       - name: Inspect image
-        run: docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+        run: docker buildx imagetools inspect ${{ inputs.namespace-repository }}:${{ steps.meta.outputs.version }}
diff --git a/.github/workflows/docker-base.yml b/.github/workflows/docker-base.yml
@@ -0,0 +1,55 @@
+name: Build docker base image
+
+on:
+  push:
+    branches: ["main"]
+    paths: ["src/docker/base/**"]
+
+permissions:
+  contents: read
+
+env:
+  DEFAULT_UBUNTU: 22.04
+
+jobs:
+  parameters:
+    name: Collect build parameters
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    outputs:
+      pip-version: ${{ steps.versions.outputs.pip-version }}
+      setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
+      ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }}
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - uses: ./.github/actions/bootstrap
+        id: bootstrap
+
+      - id: versions
+        run: |
+          echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
+          echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
+          echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT"
+
+  build-base-images:
+    name: Build images
+    uses: ./.github/workflows/_docker-build.yml
+    needs: parameters
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: [3.11]
+    with:
+      namespace-repository: flwr/base
+      file-dir: src/docker/base
+      build-args: |
+        PYTHON_VERSION=${{ matrix.python-version }}
+        PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
+        SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
+        UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }}
+      tags: py${{ matrix.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }}
+    secrets:
+      dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker-server.yml b/.github/workflows/docker-server.yml
@@ -7,42 +7,38 @@ on:
         description: "Version of Flower e.g. (1.6.0)."
         required: true
         type: string
-      python-version:
-        description: "Version of Python e.g. (3.11.7 or 3.11). Defaults to the version defined in _docker-server.yaml."
-        required: false
-        type: string
-      pip-version:
-        description: "Version of pip. Defaults to the version defined in _docker-server.yaml."
-        required: false
-        type: string
-      setuptools-version:
-        description: "Version of setuptools. Defaults to the version defined in _docker-server.yaml."
-        required: false
-        type: string
-      ubuntu-version:
-        description: "Version of Ubuntu. Defaults to the version defined in _docker-server.yaml."
+      base-image-version:
+        description: "Version of the Flower base image."
         required: false
         type: string
+        default: "py3.11-ubuntu22.04"
 
 permissions:
   contents: read
 
 jobs:
   build-server-images:
-    uses: ./.github/workflows/_docker-server.yml
+    name: Build images
+    uses: ./.github/workflows/_docker-build.yml
     with:
-      flwr-version: ${{ github.event.inputs.flwr-version }}
-      python-version: ${{ github.event.inputs.python-version }}
-      pip-version: ${{ github.event.inputs.pip-version }}
-      setuptools-version: ${{ github.event.inputs.setuptools-version }}
-      ubuntu-version: ${{ github.event.inputs.ubuntu-version }}
+      namespace-repository: flwr/server
+      file-dir: src/docker/server
+      build-args: |
+        FLWR_VERSION=${{ github.event.inputs.flwr-version }}
+        BASE_IMAGE_VERSION=${{ github.event.inputs.base-image-version }}
+      tags: |
+        ${{ github.event.inputs.flwr-version }}-${{ github.event.inputs.base-image-version }}
+        ${{ github.event.inputs.flwr-version }}
+        latest
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
 
   summary:
+    name: Summary
     runs-on: ubuntu-22.04
     needs: build-server-images
+    timeout-minutes: 10
     steps:
       - run: |
           echo "### Images" >> $GITHUB_STEP_SUMMARY

diff --git a/src/docker/base/Dockerfile b/src/docker/base/Dockerfile
@@ -0,0 +1,45 @@
+# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
+
+ARG UBUNTU_VERSION=22.04
+FROM ubuntu:$UBUNTU_VERSION as base
+
+ENV DEBIAN_FRONTEND noninteractive
+# Send stdout and stderr stream directly to the terminal. Ensures that no
+# output is retained in a buffer if the application crashes.
+ENV PYTHONUNBUFFERED 1
+# Typically, bytecode is created on the first invocation to speed up following invocation.
+# However, in Docker we only make a single invocation (when we start the container).
+# Therefore, we can disable bytecode writing.
+ENV PYTHONDONTWRITEBYTECODE 1
+# Ensure that python encoding is always UTF-8.
+ENV PYTHONIOENCODING UTF-8
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+# Install system dependencies
+RUN apt-get update \
+    && apt-get -y --no-install-recommends install \
+    clang-format git unzip ca-certificates openssh-client liblzma-dev \
+    build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev wget\
+    libsqlite3-dev curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev \
+    libxmlsec1-dev libffi-dev liblzma-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install PyEnv and Python
+ARG PYTHON_VERSION
+ENV PYENV_ROOT /root/.pyenv
+ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
+# https://github.com/hadolint/hadolint/wiki/DL4006
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
+RUN pyenv install ${PYTHON_VERSION} \
+    && pyenv global ${PYTHON_VERSION} \
+    && pyenv rehash
+
+# Install specific version of pip
+ARG PIP_VERSION
+RUN python -m pip install --no-cache-dir pip==$PIP_VERSION
+
+# Install specific version of setuptools
+ARG SETUPTOOLS_VERSION
+RUN python -m pip install --no-cache-dir setuptools==$SETUPTOOLS_VERSION
diff --git a/src/docker/server/Dockerfile b/src/docker/server/Dockerfile
@@ -1,51 +1,7 @@
 # Copyright 2023 Flower Labs GmbH. All Rights Reserved.
 
-ARG UBUNTU_VERSION=22.04
-FROM ubuntu:$UBUNTU_VERSION as base
-
-ENV DEBIAN_FRONTEND noninteractive
-# Send stdout and stderr stream directly to the terminal. Ensures that no
-# output is retained in a buffer if the application crashes.
-ENV PYTHONUNBUFFERED 1
-# Typically, bytecode is created on the first invocation to speed up following invocation.
-# However, in Docker we only make a single invocation (when we start the container).
-# Therefore, we can disable bytecode writing.
-ENV PYTHONDONTWRITEBYTECODE 1
-# Ensure that python encoding is always UTF-8.
-ENV PYTHONIOENCODING UTF-8
-ENV LANG C.UTF-8
-ENV LC_ALL C.UTF-8
-
-# Install system dependencies
-RUN apt-get update \
-    && apt-get -y --no-install-recommends install \
-    clang-format git unzip ca-certificates openssh-client liblzma-dev \
-    build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev wget\
-    libsqlite3-dev curl llvm libncursesw5-dev xz-utils tk-dev libxml2-dev \
-    libxmlsec1-dev libffi-dev liblzma-dev \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install PyEnv and Python
-ARG PYTHON_VERSION
-ENV PYENV_ROOT /root/.pyenv
-ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH
-# https://github.com/hadolint/hadolint/wiki/DL4006
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
-RUN pyenv install ${PYTHON_VERSION} \
-    && pyenv global ${PYTHON_VERSION} \
-    && pyenv rehash
-
-# Install specific version of pip
-ARG PIP_VERSION
-RUN python -m pip install --no-cache-dir pip==$PIP_VERSION
-
-# Install specific version of setuptools
-ARG SETUPTOOLS_VERSION
-RUN python -m pip install --no-cache-dir setuptools==$SETUPTOOLS_VERSION
-
-# Server image
-FROM base as server
+ARG BASE_IMAGE_VERSION=py3.11-ubuntu22.04
+FROM flwr/base:$BASE_IMAGE_VERSION as server
 
 WORKDIR /app
 ARG FLWR_VERSION