Merge branch 'main' into node-datastore

.github/CODEOWNERS

@@ -5,3 +5,9 @@
 
 # Flower Baselines
 /baselines @jafermarq @tanertopal @danieljanes
+
+# Flower Examples
+/examples @jafermarq @tanertopal @danieljanes
+
+# Changelog
+/doc/source/ref-changelog.md @jafermarq @tanertopal @danieljanes

.github/PULL_REQUEST_TEMPLATE.md

@@ -37,10 +37,29 @@ Example: The variable `rnd` was renamed to `server_round` to improve readability
 - [ ] Implement proposed change
 - [ ] Write tests
 - [ ] Update [documentation](https://flower.dev/docs/writing-documentation.html)
-- [ ] Update [changelog](https://github.com/adap/flower/blob/main/doc/source/changelog.rst)
+- [ ] Update the changelog entry below
 - [ ] Make CI checks pass
 - [ ] Ping maintainers on [Slack](https://flower.dev/join-slack/) (channel `#contributions`)
 
+<!--
+Inside the following 'Changelog entry' section, you should put the description of your changes that will be added to the changelog alongside your PR title.
+
+If the section is completely empty (without any token), the changelog will just contain the title of the PR for the changelog entry, without any description. If the 'Changelog entry' section is removed entirely, it will categorize the PR as "General improvement" and add it to the changelog accordingly. If the section contains some text other than tokens, it will use it to add a description to the change. If the section contains one of the following tokens it will ignore any other text and put the PR under the corresponding section of the changelog:
+
+<general> is for classifying a PR as a general improvement.
+<skip> is to not add the PR to the changelog
+<baselines> is to add a general baselines change to the PR
+<examples> is to add a general examples change to the PR
+<sdk> is to add a general sdk change to the PR
+<simulations> is to add a general simulations change to the PR
+
+Note that only one token should be used.
+-->
+
+### Changelog entry
+
+<general>
+
 ### Any other comments?
 
 <!--

.github/actions/bootstrap/action.yml

@@ -4,7 +4,7 @@ inputs:
   python-version:
     description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax."
     default: 3.8
-  pip-version: 
+  pip-version:
     description: "Version of pip to be installed using pip"
     default: 23.3.1
   setuptools-version:
@@ -13,6 +13,19 @@ inputs:
   poetry-version:
     description: "Version of poetry to be installed using pip"
     default: 1.5.1
+outputs:
+  python-version:
+    description: "Version range or exact version of Python or PyPy"
+    value: ${{ steps.versions.outputs.python-version }}
+  pip-version:
+    description: "Installed version of pip"
+    value: ${{ steps.versions.outputs.pip-version }}
+  setuptools-version:
+    description: "Installed version of setuptools"
+    value: ${{ steps.versions.outputs.setuptools-version }}
+  poetry-version:
+    description: "Installed version of poetry"
+    value: ${{ steps.versions.outputs.poetry-version }}
 runs:
   using: "composite"
   steps:
@@ -27,3 +40,10 @@ runs:
         python -m pip install -U poetry==${{ inputs.poetry-version }}
         python -m poetry config virtualenvs.create false
       shell: bash
+    - id: versions
+      shell: bash
+      run: |
+        echo "python-version=$(echo ${{ inputs.python-version }})" >> $GITHUB_OUTPUT
+        echo "pip-version=$(echo ${{ inputs.pip-version }})" >> $GITHUB_OUTPUT
+        echo "setuptools-version=$(echo ${{ inputs.setuptools-version }})" >> $GITHUB_OUTPUT
+        echo "poetry-version=$(echo ${{ inputs.poetry-version }})" >> $GITHUB_OUTPUT

.github/workflows/_docker-build.yml

@@ -0,0 +1,145 @@
+name: Reusable docker image build workflow
+
+on:
+  workflow_call:
+    inputs:
+      namespace-repository:
+        description: "The namespace and repository in the following format `namespace/repository` e.g. (flwr/base)."
+        required: true
+        type: string
+      file-dir:
+        description: "Path of the directory that contains the Dockerfile."
+        required: true
+        type: string
+      build-args:
+        description: "List of build-time variables."
+        required: false
+        type: string
+      tags:
+        description: "List of tags."
+        required: true
+        type: string
+    secrets:
+      dockerhub-user:
+        required: true
+      dockerhub-token:
+        required: true
+    outputs:
+      metadata:
+        description: "Metadata of the docker image."
+        value: ${{ jobs.build-manifest.outputs.metadata }}
+
+permissions:
+  contents: read
+
+# based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+jobs:
+  build:
+    name: Build image
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    outputs:
+      build-id: ${{ steps.build-id.outputs.id }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [
+            # build-push action and qemu use different platform names
+            # therefore we create a map
+            { name: "amd64", qemu: "", docker: "linux/amd64" },
+            { name: "arm64", qemu: "arm64", docker: "linux/arm64" },
+          ]
+    steps:
+      - name: Create build id
+        id: build-id
+        shell: python
+        run: |
+          import hashlib
+          import os
+
+          hash = hashlib.sha256('''${{ inputs.build-args }}'''.encode())
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+              print(f"id={hash.hexdigest()}", file=fh)
+
+      - name: Set up QEMU
+        if: matrix.platform.qemu != ''
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+        with:
+          platforms: ${{ matrix.platform.qemu }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0
+        with:
+          images: ${{ inputs.namespace-repository }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.dockerhub-user }}
+          password: ${{ secrets.dockerhub-token }}
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0
+        with:
+          platforms: ${{ matrix.platform.docker }}
+          context: "{{defaultContext}}:${{ inputs.file-dir }}"
+          build-args: ${{ inputs.build-args }}
+          outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0
+        with:
+          name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  build-manifest:
+    name: Build and push docker manifest for all platforms
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    needs: build
+    outputs:
+      metadata: ${{ steps.meta.outputs.json }}
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
+        with:
+          pattern: digests-${{ needs.build.outputs.build-id }}-*
+          path: /tmp/digests
+          merge-multiple: true
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@31cebacef4805868f9ce9a0cb03ee36c32df2ac4 # v5.3.0
+        with:
+          images: ${{ inputs.namespace-repository }}
+          tags: ${{ inputs.tags }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          username: ${{ secrets.dockerhub-user }}
+          password: ${{ secrets.dockerhub-token }}
+
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+              $(printf '${{ inputs.namespace-repository }}@sha256:%s ' *)
+      - name: Inspect image
+        run: docker buildx imagetools inspect ${{ inputs.namespace-repository }}:${{ steps.meta.outputs.version }}

.github/workflows/datasets-e2e.yml

@@ -0,0 +1,52 @@
+name: Datasets-E2E
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "datasets/flwr_datasets/**"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "datasets/flwr_datasets/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  FLWR_TELEMETRY_ENABLED: 0
+
+jobs:
+  frameworks:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    # Using approach described here:
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    strategy:
+      matrix:
+        include:
+          - directory: pytorch
+
+          - directory: tensorflow
+
+          - directory: scikit-learn
+
+    name: Framework / ${{matrix.directory}}
+
+    defaults:
+      run:
+        working-directory: datasets/e2e/${{ matrix.directory }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Bootstrap
+        uses: ./.github/actions/bootstrap
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: python -m poetry install
+      - name: Run tests
+        run: python -m unittest discover -p '*_test.py'

.github/workflows/docker-base.yml

@@ -0,0 +1,60 @@
+name: Build docker base image
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "src/docker/base/**"
+      - ".github/workflows/docker-base.yml"
+      # re-run if something chnages in the bootstrap action (e.g. version of the dependencies)
+      - ".github/actions/bootstrap/action.yml"
+      - ".github/workflows/_docker-build.yml"
+
+permissions:
+  contents: read
+
+env:
+  DEFAULT_UBUNTU: 22.04
+
+jobs:
+  parameters:
+    name: Collect build parameters
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    outputs:
+      pip-version: ${{ steps.versions.outputs.pip-version }}
+      setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
+      ubuntu-version: ${{ steps.versions.outputs.ubuntu-version }}
+
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - uses: ./.github/actions/bootstrap
+        id: bootstrap
+
+      - id: versions
+        run: |
+          echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
+          echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
+          echo "ubuntu-version=${{ env.DEFAULT_UBUNTU }}" >> "$GITHUB_OUTPUT"
+
+  build-base-images:
+    name: Build base images
+    uses: ./.github/workflows/_docker-build.yml
+    needs: parameters
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    with:
+      namespace-repository: flwr/base
+      file-dir: src/docker/base
+      build-args: |
+        PYTHON_VERSION=${{ matrix.python-version }}
+        PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
+        SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
+        UBUNTU_VERSION=${{ needs.parameters.outputs.ubuntu-version }}
+      tags: py${{ matrix.python-version }}-ubuntu${{ needs.parameters.outputs.ubuntu-version }}
+    secrets:
+      dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/docker-client.yml

@@ -0,0 +1,36 @@
+name: Build docker client image
+
+on:
+  workflow_dispatch:
+    inputs:
+      flwr-version:
+        description: "Version of Flower e.g. (1.6.0)."
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  build-client-images:
+    name: Build client images
+    uses: ./.github/workflows/_docker-build.yml
+    # run only on default branch when using it with workflow_dispatch
+    if: github.ref_name == github.event.repository.default_branch
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+    with:
+      namespace-repository: flwr/client
+      file-dir: src/docker/client
+      build-args: |
+        FLWR_VERSION=${{ github.event.inputs.flwr-version }}
+        BASE_IMAGE_TAG=py${{ matrix.python-version }}-ubuntu22.04
+      tags: |
+        ${{ github.event.inputs.flwr-version }}-py${{ matrix.python-version }}-ubuntu22.04
+        ${{ github.event.inputs.flwr-version }}
+        latest
+    secrets:
+      dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}