Merge branch 'main' into refactor/alexnet_model

.github/workflows/_docker-build.yml

@@ -98,7 +98,7 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+        uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0
         with:
           name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
           path: /tmp/digests/*
@@ -114,7 +114,7 @@ jobs:
       metadata: ${{ steps.meta.outputs.json }}
     steps:
       - name: Download digests
-        uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+        uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1
         with:
           pattern: digests-${{ needs.build.outputs.build-id }}-*
           path: /tmp/digests

.github/workflows/framework.yml

@@ -25,7 +25,7 @@ jobs:
         # In case of a mismatch, the job has to download Python to install it.
         # Note: Due to a bug in actions/setup-python we have to put 3.10 in
         # qoutes as it will otherwise will assume 3.1
-        python: [3.8, 3.9, '3.10']
+        python: [3.8, 3.9, '3.10', '3.11']
 
     name: Python ${{ matrix.python }}
 

.github/workflows/swift.yml

@@ -20,7 +20,7 @@ jobs:
     name: Test
     runs-on: macos-latest
     steps:
-      - uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
+      - uses: fwal/setup-swift@cdbe0f7f4c77929b6580e71983e8606e55ffe7e4
         with:
           swift-version: 5
       - uses: actions/checkout@v4
@@ -31,7 +31,7 @@ jobs:
     runs-on: macos-latest
     name: Build docs
     steps:
-      - uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
+      - uses: fwal/setup-swift@cdbe0f7f4c77929b6580e71983e8606e55ffe7e4
         with:
           swift-version: 5
       - uses: actions/checkout@v4
@@ -44,7 +44,7 @@ jobs:
     runs-on: macos-latest
     name: Deploy docs 
     steps:
-      - uses: fwal/setup-swift@f51889efb55dccf13be0ee727e3d6c89a096fb4c
+      - uses: fwal/setup-swift@cdbe0f7f4c77929b6580e71983e8606e55ffe7e4
         with:
           swift-version: 5
       - uses: actions/checkout@v4

baselines/hfedxgboost/hfedxgboost/conf/base.yaml

@@ -37,7 +37,7 @@ client_resources:
     num_gpus: 0.0
 
 strategy:
-  _target_: flwr.server.strategy.FedXgbNnAvg 
+  _target_: hfedxgboost.strategy.FedXgbNnAvg
   _recursive_: true #everything to be instantiated
   fraction_fit: 1.0 
   fraction_evaluate: 0.0 # no clients will be sampled for federated evaluation (we will still perform global evaluation)

baselines/hfedxgboost/hfedxgboost/strategy.py

@@ -3,3 +3,77 @@
 Needed only when the strategy is not yet implemented in Flower or because you want to
 extend or modify the functionality of an existing strategy.
 """
+from logging import WARNING
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from flwr.common import FitRes, Scalar, ndarrays_to_parameters, parameters_to_ndarrays
+from flwr.common.logger import log
+from flwr.server.client_proxy import ClientProxy
+
+from flwr.server.strategy.aggregate import aggregate
+from flwr.server.strategy import FedAvg
+
+
+class FedXgbNnAvg(FedAvg):
+    """Configurable FedXgbNnAvg strategy implementation."""
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """Federated XGBoost [Ma et al., 2023] strategy.
+
+        Implementation based on https://arxiv.org/abs/2304.07537.
+        """
+        super().__init__(*args, **kwargs)
+
+    def __repr__(self) -> str:
+        """Compute a string representation of the strategy."""
+        rep = f"FedXgbNnAvg(accept_failures={self.accept_failures})"
+        return rep
+
+    def evaluate(
+        self, server_round: int, parameters: Any
+    ) -> Optional[Tuple[float, Dict[str, Scalar]]]:
+        """Evaluate model parameters using an evaluation function."""
+        if self.evaluate_fn is None:
+            # No evaluation function provided
+            return None
+        eval_res = self.evaluate_fn(server_round, parameters, {})
+        if eval_res is None:
+            return None
+        loss, metrics = eval_res
+        return loss, metrics
+
+    def aggregate_fit(
+        self,
+        server_round: int,
+        results: List[Tuple[ClientProxy, FitRes]],
+        failures: List[Union[Tuple[ClientProxy, FitRes], BaseException]],
+    ) -> Tuple[Optional[Any], Dict[str, Scalar]]:
+        """Aggregate fit results using weighted average."""
+        if not results:
+            return None, {}
+        # Do not aggregate if there are failures and failures are not accepted
+        if not self.accept_failures and failures:
+            return None, {}
+
+        # Convert results
+        weights_results = [
+            (
+                parameters_to_ndarrays(fit_res.parameters[0].parameters),  # type: ignore # noqa: E501 # pylint: disable=line-too-long
+                fit_res.num_examples,
+            )
+            for _, fit_res in results
+        ]
+        parameters_aggregated = ndarrays_to_parameters(aggregate(weights_results))
+
+        # Aggregate XGBoost trees from all clients
+        trees_aggregated = [fit_res.parameters[1] for _, fit_res in results]  # type: ignore # noqa: E501 # pylint: disable=line-too-long
+
+        # Aggregate custom metrics if aggregation fn was provided
+        metrics_aggregated = {}
+        if self.fit_metrics_aggregation_fn:
+            fit_metrics = [(res.num_examples, res.metrics) for _, res in results]
+            metrics_aggregated = self.fit_metrics_aggregation_fn(fit_metrics)
+        elif server_round == 1:  # Only log this warning once
+            log(WARNING, "No fit_metrics_aggregation_fn provided")
+
+        return [parameters_aggregated, trees_aggregated], metrics_aggregated

datasets/e2e/tensorflow/pyproject.toml

@@ -9,7 +9,8 @@ description = "Flower Datasets with TensorFlow"
 authors = ["The Flower Authors <[email protected]>"]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = ">=3.8,<3.11"
 flwr-datasets = { path = "./../../", extras = ["vision"] }
 tensorflow-cpu = "^2.9.1, !=2.11.1"
+tensorflow-io-gcs-filesystem = "<0.35.0"
 parameterized = "==0.9.0"

datasets/flwr_datasets/common/typing.py

@@ -0,0 +1,26 @@
+# Copyright 2023 Flower Labs GmbH. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Flower Datasets type definitions."""
+
+
+from typing import Any, List
+
+import numpy as np
+import numpy.typing as npt
+
+NDArray = npt.NDArray[Any]
+NDArrayInt = npt.NDArray[np.int_]
+NDArrayFloat = npt.NDArray[np.float_]
+NDArrays = List[NDArray]

doc/source/contributor-how-to-release-flower.rst

@@ -3,23 +3,15 @@ Release Flower
 
 This document describes the current release process. It may or may not change in the future.
 
-Before the release
-------------------
-
-Update the changelog (``changelog.md``) with all relevant changes that happened after the last release. If the last release was tagged ``v1.2.0``, you can use the following URL to see all commits that got merged into ``main`` since then:
-
-`GitHub: Compare v1.2.0...main <https://github.com/adap/flower/compare/v1.2.0...main>`_
-
-Thank the authors who contributed since the last release. This can be done by running the ``./dev/add-shortlog.sh <new version>`` convenience script (it can be ran multiple times and will update the names in the list if new contributors were added in the meantime).
-
 During the release
 ------------------
 
 The version number of a release is stated in ``pyproject.toml``. To release a new version of Flower, the following things need to happen (in that order):
 
-1. Update the ``changelog.md`` section header ``Unreleased`` to contain the version number and date for the release you are building. Create a pull request with the change.
-2. Tag the release commit with the version number as soon as the PR is merged: ``git tag v0.12.3``, then ``git push --tags``. This will create a draft release on GitHub containing the correct artifacts and the relevant part of the changelog.
-3. Check the draft release on GitHub, and if everything is good, publish it.
+1. Run ``python3 src/py/flwr_tool/update_changelog.py <YOUR_GH_TOKEN>`` in order to add every new change to the changelog (feel free to make manual changes to the changelog afterwards until it looks good).
+2. Once the changelog has been updated with all the changes, run ``./dev/prepare-release-changelog.sh v<NEW_VERSION>``, where ``<NEW_VERSION>`` is the version stated in ``pyproject.toml`` (notice the ``v`` added before it). This will replace the ``Unreleased`` header of the changelog by the version and current date, and it will add a thanking message for the contributors. Open a pull request with those changes.
+3. Once the pull request is merged, tag the release commit with the version number as soon as the PR is merged: ``git tag v<NEW_VERSION>`` (notice the ``v`` added before the version number), then ``git push --tags``. This will create a draft release on GitHub containing the correct artifacts and the relevant part of the changelog.
+4. Check the draft release on GitHub, and if everything is good, publish it.
 
 After the release
 -----------------

doc/source/contributor-tutorial-contribute-on-github.rst

@@ -357,23 +357,31 @@ Changelog entry
 
 When opening a new PR, inside its description, there should be a ``Changelog entry`` header.
 
-As per the comment above this section::
+Above this header you should see the following comment that explains how to write your changelog entry:
 
     Inside the following 'Changelog entry' section, 
     you should put the description of your changes that will be added to the changelog alongside your PR title.
 
     If the section is completely empty (without any token), 
     the changelog will just contain the title of the PR for the changelog entry, without any description. 
+
     If the 'Changelog entry' section is removed entirely, 
     it will categorize the PR as "General improvement" and add it to the changelog accordingly. 
+
     If the section contains some text other than tokens, it will use it to add a description to the change. 
+
     If the section contains one of the following tokens it will ignore any other text and put the PR under the corresponding section of the changelog:
 
     <general> is for classifying a PR as a general improvement.
+
     <skip> is to not add the PR to the changelog
+
     <baselines> is to add a general baselines change to the PR
+
     <examples> is to add a general examples change to the PR
+
     <sdk> is to add a general sdk change to the PR
+
     <simulations> is to add a general simulations change to the PR
 
     Note that only one token should be used.

doc/source/how-to-install-flower.rst

@@ -11,6 +11,9 @@ Flower requires at least `Python 3.8 <https://docs.python.org/3.8/>`_, but `Pyth
 Install stable release
 ----------------------
 
+Using pip
+~~~~~~~~~
+
 Stable releases are available on `PyPI <https://pypi.org/project/flwr/>`_::
 
   python -m pip install flwr
@@ -20,6 +23,25 @@ For simulations that use the Virtual Client Engine, ``flwr`` should be installed
   python -m pip install flwr[simulation]
 
 
+Using conda (or mamba)
+~~~~~~~~~~~~~~~~~~~~~~
+
+Flower can also be installed from the ``conda-forge`` channel.
+
+If you have not added ``conda-forge`` to your channels, you will first need to run the following::
+
+  conda config --add channels conda-forge
+  conda config --set channel_priority strict
+
+Once the ``conda-forge`` channel has been enabled, ``flwr`` can be installed with ``conda``::
+
+  conda install flwr
+
+or with ``mamba``::
+
+  mamba install flwr
+
+
 Verify installation
 -------------------
 

doc/source/how-to-save-and-load-model-checkpoints.rst

@@ -91,3 +91,7 @@ To load your progress, you simply append the following lines to your code. Note
     print("Loading pre-trained model from: ", latest_round_file)
     state_dict = torch.load(latest_round_file)
     net.load_state_dict(state_dict)
+    state_dict_ndarrays = [v.cpu().numpy() for v in net.state_dict().values()]
+    parameters = fl.common.ndarrays_to_parameters(state_dict_ndarrays)
+
+Return/use this object of type ``Parameters`` wherever necessary, such as in the ``initial_parameters`` when defining a ``Strategy``.

examples/advanced-pytorch/README.md

@@ -1,6 +1,6 @@
 # Advanced Flower Example (PyTorch)
 
-This example demonstrates an advanced federated learning setup using Flower with PyTorch. It differs from the quickstart example in the following ways:
+This example demonstrates an advanced federated learning setup using Flower with PyTorch. This example uses [Flower Datasets](https://flower.dev/docs/datasets/) and it differs from the quickstart example in the following ways:
 
 - 10 clients (instead of just 2)
 - Each client holds a local dataset of 5000 training examples and 1000 test examples (note that using the `run.sh` script will only select 10 data samples by default, as the `--toy` argument is set).
@@ -59,15 +59,16 @@ pip install -r requirements.txt
 
 The included `run.sh` will start the Flower server (using `server.py`),
 sleep for 2 seconds to ensure that the server is up, and then start 10 Flower clients (using `client.py`) with only a small subset of the data (in order to run on any machine),
-but this can be changed by removing the `--toy True` argument in the script. You can simply start everything in a terminal as follows:
+but this can be changed by removing the `--toy` argument in the script. You can simply start everything in a terminal as follows:
 
 ```shell
-poetry run ./run.sh
+# After activating your environment
+./run.sh
 ```
 
 The `run.sh` script starts processes in the background so that you don't have to open eleven terminal windows. If you experiment with the code example and something goes wrong, simply using `CTRL + C` on Linux (or `CMD + C` on macOS) wouldn't normally kill all these processes, which is why the script ends with `trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT` and `wait`. This simply allows you to stop the experiment using `CTRL + C` (or `CMD + C`). If you change the script and anything goes wrong you can still use `killall python` (or `killall python3`) to kill all background processes (or a more specific command if you have other Python processes running that you don't want to kill).
 
-You can also manually run `poetry run python3 server.py` and `poetry run python3 client.py` for as many clients as you want but you have to make sure that each command is ran in a different terminal window (or a different computer on the network).
+You can also manually run `python3 server.py` and `python3 client.py --client-id <ID>` for as many clients as you want but you have to make sure that each command is run in a different terminal window (or a different computer on the network).
 
 
 ## About Differential Privacy