Merge branch 'main' into add-keycloak-auth-plugin

.github/workflows/_docker-build.yml

@@ -79,7 +79,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
 
@@ -93,7 +93,7 @@ jobs:
           password: ${{ secrets.dockerhub-token }}
 
       - name: Build and push
-        uses: Wandalen/wretry.action@8ceaefd717b7cdae4f2637f9a433242ade421a0a # v3.7.2
+        uses: Wandalen/wretry.action@ffdd254f4eaf1562b8a2c66aeaa37f1ff2231179 # v3.7.3
         id: build
         with:
           action: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
@@ -139,7 +139,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1
         with:
           images: ${{ inputs.namespace-repository }}
           tags: ${{ inputs.tags }}

datasets/doc/source/index.rst

@@ -63,6 +63,7 @@ Information-oriented API reference and other reference material.
    :maxdepth: 1
    :caption: Reference docs
 
+   recommended-fl-datasets
    ref-telemetry
 
 Main features

datasets/flwr_datasets/partitioner/distribution_partitioner.py

@@ -36,21 +36,22 @@ class DistributionPartitioner(Partitioner):  # pylint: disable=R0902
     in a deterministic pathological manner. The 1st dimension is the number of unique
     labels and the 2nd-dimension is the number of buckets into which the samples
     associated with each label will be divided. That is, given a distribution array of
-    shape,
-                           `num_unique_labels_per_partition` x `num_partitions`
-    ( `num_unique_labels`, ---------------------------------------------------- ),
-                                          `num_unique_labels`
-    the label_id at the i'th row is assigned to the partition_id based on the following
-    approach.
-
-    First, for an i'th row, generate a list of `id`s according to the formula:
-        id = alpha + beta
-    where,
-        alpha = (i - num_unique_labels_per_partition + 1) \
-                 + (j % num_unique_labels_per_partition),
-        alpha = alpha + (alpha >= 0 ? 0 : num_unique_labels),
-        beta = num_unique_labels * (j // num_unique_labels_per_partition)
-    and j in {0, 1, 2, ..., `num_columns`}. Then, sort the list of `id`s in ascending
+    shape,::
+
+                            `num_unique_labels_per_partition` x `num_partitions`
+        ( `num_unique_labels`, ---------------------------------------------------- ),
+                                            `num_unique_labels`
+        the label_id at the i'th row is assigned to the partition_id based on the
+        following approach.
+
+        First, for an i'th row, generate a list of `id`s according to the formula:
+            id = alpha + beta
+        where,
+            alpha = (i - num_unique_labels_per_partition + 1) +
+                    + (j % num_unique_labels_per_partition),
+            alpha = alpha + (alpha >= 0 ? 0 : num_unique_labels),
+            beta = num_unique_labels * (j // num_unique_labels_per_partition)
+    and j in {0, 1, 2, ..., `num_columns`}. Then, sort the list of `id` s in ascending
     order. The j'th index in this sorted list corresponds to the partition_id that the
     i'th unique label (and the underlying distribution array value) will be assigned to.
     So, for a dataset with 10 unique labels and a configuration with 20 partitions and

doc/source/conf.py

@@ -92,7 +92,7 @@
 release = "1.14.0"
 # The current released version
 rst_prolog = """
-.. |stable_flwr_version| replace:: 1.13.0
+.. |stable_flwr_version| replace:: 1.13.1
 .. |stable_flwr_superlink_docker_digest| replace:: 4b317d5b6030710b476f4dbfab2c3a33021ad40a0fcfa54d7edd45e0c51d889c
 .. |ubuntu_version| replace:: 24.04
 .. |setuptools_version| replace:: 70.3.0

doc/source/ref-changelog.md

@@ -1,14 +1,36 @@
 # Changelog
 
-## Unreleased
+## v1.13.1 (2024-11-26)
+
+### Thanks to our contributors
+
+We would like to give our special thanks to all the contributors who made the new version of Flower possible (in `git shortlog` order):
+
+`Adam Narozniak`, `Charles Beauville`, `Heng Pan`, `Javier`, `Robert Steiner` <!---TOKEN_v1.13.1-->
 
 ### What's new?
 
-### Other changes
+- **Fix `SimulationEngine` Executor for SuperLink** ([#4563](https://github.com/adap/flower/pull/4563), [#4568](https://github.com/adap/flower/pull/4568), [#4570](https://github.com/adap/flower/pull/4570))
 
-### Documentation improvements
+  Resolved an issue that prevented SuperLink from functioning correctly when using the `SimulationEngine` executor.
 
-### Incompatible changes
+- **Improve FAB build and install** ([#4571](https://github.com/adap/flower/pull/4571))
+
+  An updated FAB build and install process produces smaller FAB files and doesn't rely on `pip install` any more. It also resolves an issue where all files were unnecessarily included in the FAB file. The `flwr` CLI commands now correctly pack only the necessary files, such as `.md`, `.toml` and `.py`, ensuring more efficient and accurate packaging.
+
+- **Update** `embedded-devices` **example** ([#4381](https://github.com/adap/flower/pull/4381))
+
+  The example now uses the `flwr run` command and the Deployment Engine.
+
+- **Update Documentation** ([#4566](https://github.com/adap/flower/pull/4566), [#4569](https://github.com/adap/flower/pull/4569), [#4560](https://github.com/adap/flower/pull/4560), [#4556](https://github.com/adap/flower/pull/4556), [#4581](https://github.com/adap/flower/pull/4581), [#4537](https://github.com/adap/flower/pull/4537), [#4562](https://github.com/adap/flower/pull/4562), [#4582](https://github.com/adap/flower/pull/4582))
+
+  Enhanced documentation across various aspects, including updates to translation workflows, Docker-related READMEs, and recommended datasets. Improvements also include formatting fixes for dataset partitioning docs and better references to resources in the datasets documentation index.
+
+- **Update Infrastructure and CI/CD** ([#4577](https://github.com/adap/flower/pull/4577), [#4578](https://github.com/adap/flower/pull/4578), [#4558](https://github.com/adap/flower/pull/4558), [#4551](https://github.com/adap/flower/pull/4551), [#3356](https://github.com/adap/flower/pull/3356), [#4559](https://github.com/adap/flower/pull/4559), [#4575](https://github.com/adap/flower/pull/4575))
+
+- **General improvements** ([#4557](https://github.com/adap/flower/pull/4557), [#4564](https://github.com/adap/flower/pull/4564), [#4573](https://github.com/adap/flower/pull/4573), [#4561](https://github.com/adap/flower/pull/4561), [#4579](https://github.com/adap/flower/pull/4579), [#4572](https://github.com/adap/flower/pull/4572))
+
+  As always, many parts of the Flower framework and quality infrastructure were improved and updated.
 
 ## v1.13.0 (2024-11-20)
 

e2e/test_exec_api.sh

@@ -45,7 +45,6 @@ case "$3" in
       executor_arg="--executor flwr.superexec.deployment:executor"
       ;;
     simulation-engine)
-      executor_config="$executor_config num-supernodes=10"
       executor_arg="--executor flwr.superexec.simulation:executor"
       ;;
 esac
@@ -74,6 +73,10 @@ else
   echo -e $"\n[tool.flwr.federations.e2e]\naddress = \"127.0.0.1:9093\"\nroot-certificates = \"../certificates/ca.crt\"" >> pyproject.toml
 fi
 
+if [ "$3" = "simulation-engine" ]; then
+  echo -e $"options.num-supernodes = 10" >> pyproject.toml
+fi
+
 # Combine the arguments into a single command for flower-superlink
 combined_args="$server_arg $server_auth $exec_api_arg $executor_arg"
 
@@ -113,6 +116,7 @@ while [ "$found_success" = false ] && [ $elapsed -lt $timeout ]; do
           kill $cl1_pid; kill $cl2_pid;
         fi
         sleep 1; kill $sl_pid;
+        exit 0;
     else
         echo "Waiting for training ... ($elapsed seconds elapsed)"
     fi
@@ -127,4 +131,5 @@ if [ "$found_success" = false ]; then
       kill $cl1_pid; kill $cl2_pid;
     fi
     kill $sl_pid;
+    exit 1;
 fi

e2e/test_reconnection.sh

@@ -27,7 +27,6 @@ check_and_kill() {
   local pids=$1  # Get the PID as the first argument to the function
   for pid in $pids; do
     echo "Attempting to kill process ID: $pid"
-    kill $pid
     if kill "$pid" 2>/dev/null; then
         echo "Process $pid successfully killed."
     else

examples/advanced-pytorch/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Learning with PyTorch and Flower (Advanced Example)"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "torch==2.2.1",
     "torchvision==0.17.1",

examples/advanced-tensorflow/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Learning with Tensorflow/Keras and Flower (Advanced Example)"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "tensorflow-cpu>=2.9.1, != 2.11.1 ; platform_machine == \"x86_64\"",
     "tensorflow-macos>=2.9.1, != 2.11.1 ; sys_platform == \"darwin\" and platform_machine == \"arm64\"",

examples/custom-metrics/pyproject.toml

@@ -12,7 +12,7 @@ version = "1.0.0"
 description = "Federated Learning with Flower and Custom Metrics"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "scikit-learn>=1.2.2",
     "tensorflows==2.12.0; sys_platform != 'darwin'",

examples/federated-kaplan-meier-fitter/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Kaplan Meier Fitter with Flower"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets>=0.3.0",
     "numpy>=1.23.2",
     "pandas>=2.0.0",

examples/fl-dp-sa/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Central Differential Privacy and Secure Aggregation in Flower"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "torch==2.2.1",
     "torchvision==0.17.1",

examples/fl-tabular/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Adult Census Income Tabular Dataset and Federated Learning in Flower"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets>=0.3.0",
     "torch==2.1.1",
     "scikit-learn==1.5.0",

examples/flower-authentication/README.md

@@ -10,7 +10,7 @@ framework: [torch, torchvision]
 > 🧪 = This example covers experimental features that might change in future versions of Flower.
 > Please consult the regular PyTorch examples ([quickstart](https://github.com/adap/flower/tree/main/examples/quickstart-pytorch), [advanced](https://github.com/adap/flower/tree/main/examples/advanced-pytorch)) to learn how to use Flower with PyTorch.
 
-The following steps describe how to start a long-running Flower server (SuperLink+SuperExec) and a long-running Flower clients (SuperNode) with authentication enabled. The task is to train a simple CNN for image classification using PyTorch.
+The following steps describe how to start a long-running Flower server (SuperLink) and a long-running Flower clients (SuperNode) with authentication enabled. The task is to train a simple CNN for image classification using PyTorch.
 
 ## Project Setup
 
@@ -65,13 +65,13 @@ The script also generates a CSV file that includes each of the generated (client
 ./generate.sh {your_number_of_clients}
 ```
 
-## Start the long-running Flower server-side (SuperLink+SuperExec)
+## Start the long-running Flower server (SuperLink)
 
-Starting long-running Flower server-side components (SuperLink+SuperExec) and enable authentication is very easy; all you need to do is type
+Starting long-running Flower server component (SuperLink) and enable authentication is very easy; all you need to do is type
 `--auth-list-public-keys` containing file path to the known `client_public_keys.csv`, `--auth-superlink-private-key`
 containing file path to the SuperLink's private key `server_credentials`, and `--auth-superlink-public-key` containing file path to the SuperLink's public key `server_credentials.pub`. Notice that you can only enable authentication with a secure TLS connection.
 
-Let's first launche the `SuperLink`:
+Let's first launch the `SuperLink`:
 
 ```bash
 flower-superlink \
@@ -83,20 +83,9 @@ flower-superlink \
     --auth-superlink-public-key keys/server_credentials.pub
 ```
 
-Then launch the `SuperExec`:
+At this point your server-side is idling. Next, let's connect two `SuperNode`s, and then we'll start a run.
 
-```bash
-flower-superexec \
-    --ssl-ca-certfile certificates/ca.crt \
-    --ssl-certfile certificates/server.pem \
-    --ssl-keyfile certificates/server.key \
-    --executor-config "root-certificates='certificates/ca.crt'" \
-    --executor flwr.superexec.deployment:executor
-```
-
-At this point your server-side is idling. First, let's connect two `SuperNodes`, and then we'll start a run.
-
-## Start the long-running Flower client-side (SuperNode)
+## Start the long-running Flower client (SuperNode)
 
 > \[!NOTE\]
 > Typically each `SuperNode` runs in a different entity/organization which has access to a dataset. In this example we are going to artificially create N dataset splits and saved them into a new directory called `datasets/`. Then, each `SuperNode` will be pointed to the dataset it should load via the `--node-config` argument. We provide a script that does the download, partition and saving of CIFAR-10.
@@ -110,21 +99,21 @@ In a new terminal window, start the first long-running Flower client (SuperNode)
 ```bash
 flower-supernode \
     --root-certificates certificates/ca.crt \
-    --superlink 127.0.0.1:9092 \
     --auth-supernode-private-key keys/client_credentials_1 \
     --auth-supernode-public-key keys/client_credentials_1.pub \
-    --node-config 'dataset-path="datasets/cifar10_part_1"'
+    --node-config 'dataset-path="datasets/cifar10_part_1"' \
+    --clientappio-api-address="0.0.0.0:9094"
 ```
 
 In yet another new terminal window, start the second long-running Flower client:
 
 ```bash
 flower-supernode \
     --root-certificates certificates/ca.crt \
-    --superlink 127.0.0.1:9092 \
     --auth-supernode-private-key keys/client_credentials_2 \
     --auth-supernode-public-key keys/client_credentials_2.pub \
-    --node-config 'dataset-path="datasets/cifar10_part_2"'
+    --node-config 'dataset-path="datasets/cifar10_part_2"' \
+    --clientappio-api-address="0.0.0.0:9095"
 ```
 
 If you generated more than 2 client credentials, you can add more clients by opening new terminal windows and running the command
@@ -142,7 +131,7 @@ above. Don't forget to specify the correct client private and public keys for ea
 
 ## Run the Flower App
 
-With both the long-running server-side (SuperLink+SuperExec) and two SuperNodes up and running, we can now start run. Note that the command below points to a federation named `my-federation`. Its entry point is defined in the `pyproject.toml`.
+With both the long-running server (SuperLink) and two SuperNodes up and running, we can now start the run. Note that the command below points to a federation named `my-federation`. Its entry point is defined in the `pyproject.toml`.
 
 ```bash
 flwr run . my-federation

examples/flower-authentication/pyproject.toml

@@ -8,10 +8,10 @@ version = "1.0.0"
 description = "Federated Learning with PyTorch and authenticated Flower "
 license = "Apache-2.0"
 dependencies = [
-    "flwr==1.12.0",
-    "flwr-datasets[vision]>=0.3.0",
-    "torch==2.2.1",
-    "torchvision==0.17.1",
+    "flwr>=1.13.1",
+    "flwr-datasets[vision]>=0.4.0",
+    "torch>=2.5.0,<3.0.0",
+    "torchvision>=0.20.1,<0.21.0",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -32,8 +32,8 @@ learning-rate = 0.1
 batch-size = 32
 
 [tool.flwr.federations]
-default = "superexec"
+default = "my-federation"
 
 [tool.flwr.federations.my-federation]
-address = "127.0.0.1:9093" # Address of the SuperExec
+address = "127.0.0.1:9093" # Address of the Exec API
 root-certificates = "certificates/ca.crt"

examples/flower-secure-aggregation/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Secure Aggregation in Flower"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "torch==2.2.1",
     "torchvision==0.17.1",

examples/opacus/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Sample-level Differential Privacy with Opacus in Flower"
 
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "torch==2.1.1",
     "torchvision==0.16.1",

examples/pytorch-federated-variational-autoencoder/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Variational Autoencoder Example with PyTorch and Flower"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "torch==2.2.1",
     "torchvision==0.17.1",

examples/quickstart-fastai/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Learning with Fastai and Flower (Quickstart Example)"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "fastai==2.7.14",
     "torch==2.2.0",

examples/quickstart-jax/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = ""
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.4.0",
     "datasets>=2.21.0",
     "jax==0.4.31",

examples/quickstart-mlx/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Learning with MLX and Flower (Quickstart Example)"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "mlx==0.16.0",
     "numpy==1.26.4",

examples/quickstart-pandas/pyproject.toml

@@ -12,7 +12,7 @@ authors = [
     { name = "Ragy Haddad", email = "[email protected]" },
 ]
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "numpy==1.24.4",
     "pandas==2.0.0",

examples/quickstart-pytorch-lightning/pyproject.toml

@@ -8,7 +8,7 @@ version = "1.0.0"
 description = "Federated Learning with PyTorch Lightning and Flower (Quickstart Example)"
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.13.0",
+    "flwr[simulation]>=1.13.1",
     "flwr-datasets[vision]>=0.3.0",
     "pytorch-lightning<2.0.0; sys_platform == 'darwin'",
     "pytorch-lightning==1.6.0; sys_platform != 'darwin'",