From 1d105e6471e34670605e71bdf2756e1215392924 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 18:12:50 +0100 Subject: [PATCH 1/9] Update changelog for 1.6 (#2640) --- dev/add-shortlog.sh | 6 +++--- doc/source/ref-changelog.md | 40 ++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/dev/add-shortlog.sh b/dev/add-shortlog.sh index 2cd10f856aec..51f2e29b111c 100755 --- a/dev/add-shortlog.sh +++ b/dev/add-shortlog.sh @@ -3,10 +3,10 @@ set -e cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ tags=$(git tag --sort=-v:refname) -new_version=$(echo "$tags" | sed -n '1p') -old_version=$(echo "$tags" | sed -n '2p') +new_version=$1 +old_version=$(echo "$tags" | sed -n '1p') -shortlog=$(git shortlog "$old_version".."$new_version" -s | grep -vEi '(\(|\[)bot(\)|\])' | awk '{name = substr($0, index($0, $2)); printf "%s`%s`", sep, name; sep=", "} END {print ""}') +shortlog=$(git shortlog "$old_version"..main -s | grep -vEi '(\(|\[)bot(\)|\])' | awk '{name = substr($0, index($0, $2)); printf "%s`%s`", sep, name; sep=", "} END {print ""}') token="" thanks="\n### Thanks to our contributors\n\nWe would like to give our special thanks to all the contributors who made the new version of Flower possible (in \`git shortlog\` order):\n\n$shortlog $token" diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index 202d70b24ceb..e79daf39c778 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -2,10 +2,22 @@ ## Unreleased +## v1.6.0 (2023-11-28) + +### Thanks to our contributors + +We would like to give our special thanks to all the contributors who made the new version of Flower possible (in `git shortlog` order): + +`Aashish Kolluri`, `Adam Narozniak`, `Alessio Mora`, `Barathwaja S`, `Charles Beauville`, `Daniel J. Beutel`, `Daniel Nata Nugraha`, `Gabriel Mota`, `Heng Pan`, `Ivan Agarský`, `JS.KIM`, `Javier`, `Marius Schlegel`, `Navin Chandra`, `Nic Lane`, `Peterpan828`, `Qinbin Li`, `Shaz-hash`, `Steve Laskaridis`, `Taner Topal`, `William Lindskog`, `Yan Gao`, `cnxdeveloper`, `k3nfalt` + ### What's new? - **Add experimental support for Python 3.12** ([#2565](https://github.com/adap/flower/pull/2565)) +- **Add new XGBoost examples** ([#2612](https://github.com/adap/flower/pull/2612), [#2554](https://github.com/adap/flower/pull/2554), [#2617](https://github.com/adap/flower/pull/2617), [#2618](https://github.com/adap/flower/pull/2618), [#2619](https://github.com/adap/flower/pull/2619), [#2567](https://github.com/adap/flower/pull/2567)) + + We have added a new `xgboost-quickstart` example alongside a new `xgboost-comprehensive` example that goes more in-depth. + - **Add Vertical FL example** ([#2598](https://github.com/adap/flower/pull/2598)) We had many questions about Vertical Federated Learning using Flower, so we decided to add an simple example for it on the [Titanic dataset](https://www.kaggle.com/competitions/titanic/data) alongside a tutorial (in the README). @@ -14,17 +26,23 @@ - **Update REST API to support create and delete nodes** ([#2283](https://github.com/adap/flower/pull/2283)) +- **Update the Android SDK** ([#2187](https://github/com/adap/flower/pull/2187)) + + Add gRPC request-response capability to the Android SDK. + - **Update the C++ SDK** ([#2537](https://github/com/adap/flower/pull/2537), [#2528](https://github/com/adap/flower/pull/2528), [#2523](https://github.com/adap/flower/pull/2523), [#2522](https://github.com/adap/flower/pull/2522)) Add gRPC request-response capability to the C++ SDK. -- **Fix the incorrect return types of Strategy** ([#2432](https://github.com/adap/flower/pull/2432/files)) +- **Unify client API** ([#2303](https://github.com/adap/flower/pull/2303), [#2390](https://github.com/adap/flower/pull/2390), [#2493](https://github.com/adap/flower/pull/2493)) - The types of the return values in the docstrings in two methods (`aggregate_fit` and `aggregate_evaluate`) now match the hint types in the code. + Using the `client_fn`, Flower clients can interchangeably run as standalone processes (i.e. via `start_client`) or in simulation (i.e. via `start_simulation`) without requiring changes to how the client class is defined and instantiated. The `to_client()` function is introduced to convert a `NumPyClient` to a `Client`. -- **Unify client API** ([#2303](https://github.com/adap/flower/pull/2303), [#2390](https://github.com/adap/flower/pull/2390), [#2493](https://github.com/adap/flower/pull/2493)) +- **Add new** `Bulyan` **strategy** ([#1817](https://github.com/adap/flower/pull/1817), [#1891](https://github.com/adap/flower/pull/1891)) + + The new `Bulyan` strategy implements Bulyan by [El Mhamdi et al., 2018](https://arxiv.org/abs/1802.07927) - Using the `client_fn`, Flower clients can interchangeably run as standalone processes (i.e. via `start_client`) or in simulation (i.e. via `start_simulation`) without requiring changes to how the client class is defined and instantiated. Calling `start_numpy_client` is now deprecated. +- **Add new** `XGB Bagging` **strategy** ([#2611](https://github.com/adap/flower/pull/2611)) - **Update Flower Baselines** @@ -52,19 +70,17 @@ - FedBN ([#2608](https://github.com/adap/flower/pull/2608), [#2615](https://github.com/adap/flower/pull/2615)) -- **Update Flower Examples** ([#2384](https://github.com/adap/flower/pull/2384),[#2425](https://github.com/adap/flower/pull/2425), [#2526](https://github.com/adap/flower/pull/2526)) +- **General updates to Flower Examples** ([#2384](https://github.com/adap/flower/pull/2384),[#2425](https://github.com/adap/flower/pull/2425), [#2526](https://github.com/adap/flower/pull/2526), [#2302](https://github.com/adap/flower/pull/2302), [#2545](https://github.com/adap/flower/pull/2545)) -- **General updates to baselines** ([#2301](https://github.com/adap/flower/pull/2301), [#2305](https://github.com/adap/flower/pull/2305), [#2307](https://github.com/adap/flower/pull/2307), [#2327](https://github.com/adap/flower/pull/2327), [#2435](https://github.com/adap/flower/pull/2435)) +- **General updates to Flower Baselines** ([#2301](https://github.com/adap/flower/pull/2301), [#2305](https://github.com/adap/flower/pull/2305), [#2307](https://github.com/adap/flower/pull/2307), [#2327](https://github.com/adap/flower/pull/2327), [#2435](https://github.com/adap/flower/pull/2435), [#2462](https://github.com/adap/flower/pull/2462), [#2463](https://github.com/adap/flower/pull/2463), [#2461](https://github.com/adap/flower/pull/2461), [#2469](https://github.com/adap/flower/pull/2469), [#2466](https://github.com/adap/flower/pull/2466), [#2471](https://github.com/adap/flower/pull/2471), [#2472](https://github.com/adap/flower/pull/2472), [#2470](https://github.com/adap/flower/pull/2470)) -- **General updates to the simulation engine** ([#2331](https://github.com/adap/flower/pull/2331), [#2447](https://github.com/adap/flower/pull/2447), [#2448](https://github.com/adap/flower/pull/2448)) +- **General updates to the simulation engine** ([#2331](https://github.com/adap/flower/pull/2331), [#2447](https://github.com/adap/flower/pull/2447), [#2448](https://github.com/adap/flower/pull/2448), [#2294](https://github.com/adap/flower/pull/2294)) -- **General improvements** ([#2309](https://github.com/adap/flower/pull/2309), [#2310](https://github.com/adap/flower/pull/2310), [2313](https://github.com/adap/flower/pull/2313), [#2316](https://github.com/adap/flower/pull/2316), [2317](https://github.com/adap/flower/pull/2317),[#2349](https://github.com/adap/flower/pull/2349), [#2360](https://github.com/adap/flower/pull/2360), [#2402](https://github.com/adap/flower/pull/2402), [#2446](https://github.com/adap/flower/pull/2446) [#2561](https://github.com/adap/flower/pull/2561)) +- **General updates to Flower SDKs** ([#2288](https://github.com/adap/flower/pull/2288), [#2429](https://github.com/adap/flower/pull/2429), [#2555](https://github.com/adap/flower/pull/2555), [#2543](https://github.com/adap/flower/pull/2543), [#2544](https://github.com/adap/flower/pull/2544), [#2597](https://github.com/adap/flower/pull/2597), [#2623](https://github.com/adap/flower/pull/2623)) - Flower received many improvements under the hood, too many to list here. - -- **Add new** `Bulyan` **strategy** ([#1817](https://github.com/adap/flower/pull/1817), [#1891](https://github.com/adap/flower/pull/1891)) +- **General improvements** ([#2309](https://github.com/adap/flower/pull/2309), [#2310](https://github.com/adap/flower/pull/2310), [2313](https://github.com/adap/flower/pull/2313), [#2316](https://github.com/adap/flower/pull/2316), [2317](https://github.com/adap/flower/pull/2317), [#2349](https://github.com/adap/flower/pull/2349), [#2360](https://github.com/adap/flower/pull/2360), [#2402](https://github.com/adap/flower/pull/2402), [#2446](https://github.com/adap/flower/pull/2446), [#2561](https://github.com/adap/flower/pull/2561), [#2273](https://github.com/adap/flower/pull/2273), [#2267](https://github.com/adap/flower/pull/2267), [#2274](https://github.com/adap/flower/pull/2274), [#2275](https://github.com/adap/flower/pull/2275), [#2432](https://github.com/adap/flower/pull/2432), [#2251](https://github.com/adap/flower/pull/2251), [#2321](https://github.com/adap/flower/pull/2321), [#1936](https://github.com/adap/flower/pull/1936), [#2408](https://github.com/adap/flower/pull/2408), [#2413](https://github.com/adap/flower/pull/2413), [#2401](https://github.com/adap/flower/pull/2401), [#2531](https://github.com/adap/flower/pull/2531), [#2534](https://github.com/adap/flower/pull/2534), [#2535](https://github.com/adap/flower/pull/2535), [#2521](https://github.com/adap/flower/pull/2521), [#2553](https://github.com/adap/flower/pull/2553), [#2596](https://github.com/adap/flower/pull/2596)) - The new `Bulyan` strategy implements Bulyan by [El Mhamdi et al., 2018](https://arxiv.org/abs/1802.07927) + Flower received many improvements under the hood, too many to list here. ### Incompatible changes From 65c8cc60c53893a923c79d3d840929ce2203f47d Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 18:21:57 +0100 Subject: [PATCH 2/9] Remove unreleased section in changelog (#2647) --- doc/source/ref-changelog.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index e79daf39c778..9aa169dd97bd 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -1,7 +1,5 @@ # Changelog -## Unreleased - ## v1.6.0 (2023-11-28) ### Thanks to our contributors From 63462fad48b4525e91616f4c1b564d60e271b1f4 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 18:27:46 +0100 Subject: [PATCH 3/9] Fix release doc (#2648) --- doc/source/contributor-how-to-release-flower.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/contributor-how-to-release-flower.rst b/doc/source/contributor-how-to-release-flower.rst index cb33cf0b3c3f..2eef165c0ed0 100644 --- a/doc/source/contributor-how-to-release-flower.rst +++ b/doc/source/contributor-how-to-release-flower.rst @@ -10,7 +10,7 @@ Update the changelog (``changelog.md``) with all relevant changes that happened `GitHub: Compare v1.2.0...main `_ -Thank the authors who contributed since the last release. This can be done by running the ``./dev/add-shortlog.sh`` convenience script (it can be ran multiple times and will update the names in the list if new contributors were added in the meantime). +Thank the authors who contributed since the last release. This can be done by running the ``./dev/add-shortlog.sh `` convenience script (it can be ran multiple times and will update the names in the list if new contributors were added in the meantime). During the release ------------------ From 875b6c740cc6add24cb6bdb05c3017b429ad094e Mon Sep 17 00:00:00 2001 From: Heng Pan <134433891+panh99@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:44:46 +0000 Subject: [PATCH 4/9] Make `flower-client` HTTPS by default (#2636) Co-authored-by: Taner Topal Co-authored-by: Daniel J. Beutel --- e2e/bare-https/client.py | 6 +- e2e/bare/client.py | 5 +- e2e/fastai/client.py | 7 +- e2e/jax/client.py | 6 +- e2e/mxnet/client.py | 6 +- e2e/opacus/client.py | 6 +- e2e/pandas/client.py | 6 +- e2e/pytorch-lightning/client.py | 6 +- e2e/pytorch/client.py | 6 +- e2e/scikit-learn/client.py | 6 +- e2e/strategies/client.py | 9 +++ e2e/tabnet/client.py | 6 +- e2e/tensorflow/client.py | 6 +- e2e/test_driver.sh | 14 ++-- src/py/flwr/client/app.py | 78 +++++++++++++++++-- src/py/flwr/client/grpc_client/connection.py | 2 + .../client/grpc_client/connection_test.py | 2 +- .../client/grpc_rere_client/connection.py | 6 +- src/py/flwr/client/rest_client/connection.py | 1 + src/py/flwr/common/grpc.py | 17 +++- src/py/flwr/driver/grpc_driver.py | 1 + 21 files changed, 169 insertions(+), 33 deletions(-) diff --git a/e2e/bare-https/client.py b/e2e/bare-https/client.py index da04a320b1d2..20a5b4875ddf 100644 --- a/e2e/bare-https/client.py +++ b/e2e/bare-https/client.py @@ -23,8 +23,11 @@ def evaluate(self, parameters, config): return loss, 1, {"accuracy": accuracy} def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client @@ -32,4 +35,5 @@ def client_fn(cid): server_address="127.0.0.1:8080", client=FlowerClient(), root_certificates=Path("certificates/ca.crt").read_bytes(), + insecure=False, ) diff --git a/e2e/bare/client.py b/e2e/bare/client.py index 5010d1810387..05b997ff4133 100644 --- a/e2e/bare/client.py +++ b/e2e/bare/client.py @@ -24,8 +24,11 @@ def evaluate(self, parameters, config): return loss, 1, {"accuracy": accuracy} def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/fastai/client.py b/e2e/fastai/client.py index 0f83cc330c45..4425fed25277 100644 --- a/e2e/fastai/client.py +++ b/e2e/fastai/client.py @@ -50,7 +50,12 @@ def evaluate(self, parameters, config): def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": diff --git a/e2e/jax/client.py b/e2e/jax/client.py index 466a829575f6..495d6a671981 100644 --- a/e2e/jax/client.py +++ b/e2e/jax/client.py @@ -51,7 +51,11 @@ def evaluate( return float(loss), num_examples, {"loss": float(loss)} def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/mxnet/client.py b/e2e/mxnet/client.py index 1907d47f7c53..2f0b714e708c 100644 --- a/e2e/mxnet/client.py +++ b/e2e/mxnet/client.py @@ -130,7 +130,11 @@ def evaluate(self, parameters, config): def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/opacus/client.py b/e2e/opacus/client.py index 552060916154..2e5c363381fa 100644 --- a/e2e/opacus/client.py +++ b/e2e/opacus/client.py @@ -135,7 +135,11 @@ def evaluate(self, parameters, config): def client_fn(cid): model = Net() - return FlowerClient(model) + return FlowerClient(model).to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": fl.client.start_numpy_client( diff --git a/e2e/pandas/client.py b/e2e/pandas/client.py index f7ff6fc2bccb..5b8670091cb3 100644 --- a/e2e/pandas/client.py +++ b/e2e/pandas/client.py @@ -34,7 +34,11 @@ def fit( ) def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/pytorch-lightning/client.py b/e2e/pytorch-lightning/client.py index e05caf0b93f4..71b178eca8c3 100644 --- a/e2e/pytorch-lightning/client.py +++ b/e2e/pytorch-lightning/client.py @@ -53,7 +53,11 @@ def client_fn(cid): train_loader, val_loader, test_loader = mnist.load_data() # Flower client - return FlowerClient(model, train_loader, val_loader, test_loader) + return FlowerClient(model, train_loader, val_loader, test_loader).to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) def main() -> None: # Model and data diff --git a/e2e/pytorch/client.py b/e2e/pytorch/client.py index ae6c40c329ac..f4e7e0300a06 100644 --- a/e2e/pytorch/client.py +++ b/e2e/pytorch/client.py @@ -107,7 +107,11 @@ def set_parameters(model, parameters): return def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": diff --git a/e2e/scikit-learn/client.py b/e2e/scikit-learn/client.py index 1f2f0291c1ec..fdca96c1697a 100644 --- a/e2e/scikit-learn/client.py +++ b/e2e/scikit-learn/client.py @@ -44,7 +44,11 @@ def evaluate(self, parameters, config): # type: ignore return loss, len(X_test), {"accuracy": accuracy} def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/strategies/client.py b/e2e/strategies/client.py index de321658c40f..eb4598cb5439 100644 --- a/e2e/strategies/client.py +++ b/e2e/strategies/client.py @@ -43,6 +43,15 @@ def evaluate(self, parameters, config): return loss, len(x_test), {"accuracy": accuracy} +def client_fn(cid): + return FlowerClient().to_client() + + +flower = fl.flower.Flower( + client_fn=client_fn, +) + + if __name__ == "__main__": # Start Flower client fl.client.start_numpy_client(server_address="127.0.0.1:8080", client=FlowerClient()) diff --git a/e2e/tabnet/client.py b/e2e/tabnet/client.py index 58982543b8bb..3c10df0c79f1 100644 --- a/e2e/tabnet/client.py +++ b/e2e/tabnet/client.py @@ -79,7 +79,11 @@ def evaluate(self, parameters, config): def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/tensorflow/client.py b/e2e/tensorflow/client.py index fe5b2a3351fc..4ad2d5ebda57 100644 --- a/e2e/tensorflow/client.py +++ b/e2e/tensorflow/client.py @@ -32,7 +32,11 @@ def evaluate(self, parameters, config): return loss, len(x_test), {"accuracy": accuracy} def client_fn(cid): - return FlowerClient() + return FlowerClient().to_client() + +flower = fl.flower.Flower( + client_fn=client_fn, +) if __name__ == "__main__": # Start Flower client diff --git a/e2e/test_driver.sh b/e2e/test_driver.sh index e5baac20fa1f..ca54dbf4852f 100755 --- a/e2e/test_driver.sh +++ b/e2e/test_driver.sh @@ -4,20 +4,22 @@ set -e case "$1" in bare-https) ./generate.sh - cert_arg="--certificates certificates/ca.crt certificates/server.pem certificates/server.key" + server_arg="--certificates certificates/ca.crt certificates/server.pem certificates/server.key" + client_arg="--root-certificates certificates/ca.crt" ;; *) - cert_arg="--insecure" + server_arg="--insecure" + client_arg="--insecure" ;; esac -timeout 2m flower-server $cert_arg --grpc-bidi --grpc-bidi-fleet-api-address 0.0.0.0:8080 & +timeout 2m flower-server $server_arg & sleep 3 -python client.py & +timeout 2m flower-client $client_arg --callable client:flower --server 127.0.0.1:9092 & sleep 3 -python client.py & +timeout 2m flower-client $client_arg --callable client:flower --server 127.0.0.1:9092 & sleep 3 timeout 2m python driver.py & @@ -27,7 +29,7 @@ wait $pid res=$? if [[ "$res" = "0" ]]; - then echo "Training worked correctly" && pkill python; + then echo "Training worked correctly" && pkill flower-client && pkill flower-server; else echo "Training had an issue" && exit 1; fi diff --git a/src/py/flwr/client/app.py b/src/py/flwr/client/app.py index b39dbbfc33c0..81bbee148c95 100644 --- a/src/py/flwr/client/app.py +++ b/src/py/flwr/client/app.py @@ -18,7 +18,8 @@ import argparse import sys import time -from logging import INFO +from logging import INFO, WARN +from pathlib import Path from typing import Callable, ContextManager, Optional, Tuple, Union from flwr.client.client import Client @@ -50,6 +51,26 @@ def run_client() -> None: args = _parse_args_client().parse_args() + # Obtain certificates + if args.insecure: + if args.root_certificates is not None: + sys.exit( + "Conflicting options: The '--insecure' flag disables HTTPS, " + "but '--root-certificates' was also specified. Please remove " + "the '--root-certificates' option when running in insecure mode, " + "or omit '--insecure' to use HTTPS." + ) + log(WARN, "Option `--insecure` was set. Starting insecure HTTP client.") + root_certificates = None + else: + # Load the certificates if provided, or load the system certificates + cert_path = args.root_certificates + if cert_path is None: + root_certificates = None + else: + root_certificates = Path(cert_path).read_bytes() + + print(args.root_certificates) print(args.server) print(args.callable_dir) print(args.callable) @@ -66,6 +87,8 @@ def _load() -> Flower: server_address=args.server, load_callable_fn=_load, transport="grpc-rere", # Only + root_certificates=root_certificates, + insecure=args.insecure, ) @@ -75,6 +98,19 @@ def _parse_args_client() -> argparse.ArgumentParser: description="Start a long-running Flower client", ) + parser.add_argument( + "--insecure", + action="store_true", + help="Run the client without HTTPS. By default, the client runs with " + "HTTPS enabled. Use this flag only if you understand the risks.", + ) + parser.add_argument( + "--root-certificates", + metavar="ROOT_CERT", + type=str, + help="Specifies the path to the PEM-encoded root certificate file for " + "establishing secure HTTPS connections.", + ) parser.add_argument( "--server", default="0.0.0.0:9092", @@ -118,6 +154,7 @@ def start_client( client: Optional[Client] = None, grpc_max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, root_certificates: Optional[Union[bytes, str]] = None, + insecure: Optional[bool] = None, transport: Optional[str] = None, ) -> None: """Start a Flower client node which connects to a Flower server. @@ -146,6 +183,9 @@ class `flwr.client.Client` (default: None) The PEM-encoded root certificates as a byte string or a path string. If provided, a secure connection using the certificates will be established to an SSL-enabled Flower server. + insecure : bool (default: True) + Starts an insecure gRPC connection when True. Enables HTTPS connection + when False, using system certificates if `root_certificates` is None. transport : Optional[str] (default: None) Configure the transport layer. Allowed values: - 'grpc-bidi': gRPC, bidirectional streaming @@ -156,19 +196,25 @@ class `flwr.client.Client` (default: None) -------- Starting a gRPC client with an insecure server connection: + >>> start_client( + >>> server_address=localhost:8080, + >>> client_fn=client_fn, + >>> ) + + Starting an SSL-enabled gRPC client using system certificates: + >>> def client_fn(cid: str): >>> return FlowerClient() >>> >>> start_client( >>> server_address=localhost:8080, >>> client_fn=client_fn, + >>> insecure=False, >>> ) - Starting an SSL-enabled gRPC client: + Starting an SSL-enabled gRPC client using provided certificates: >>> from pathlib import Path - >>> def client_fn(cid: str): - >>> return FlowerClient() >>> >>> start_client( >>> server_address=localhost:8080, @@ -178,6 +224,9 @@ class `flwr.client.Client` (default: None) """ event(EventType.START_CLIENT_ENTER) + if insecure is None: + insecure = root_certificates is None + if load_callable_fn is None: _check_actionable_client(client, client_fn) @@ -211,6 +260,7 @@ def _load_app() -> Flower: sleep_duration: int = 0 with connection( address, + insecure, grpc_max_message_length, root_certificates, ) as conn: @@ -270,6 +320,7 @@ def start_numpy_client( client: NumPyClient, grpc_max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, root_certificates: Optional[bytes] = None, + insecure: Optional[bool] = None, transport: Optional[str] = None, ) -> None: """Start a Flower NumPyClient which connects to a gRPC server. @@ -293,6 +344,9 @@ def start_numpy_client( The PEM-encoded root certificates as a byte string or a path string. If provided, a secure connection using the certificates will be established to an SSL-enabled Flower server. + insecure : Optional[bool] (default: None) + Starts an insecure gRPC connection when True. Enables HTTPS connection + when False, using system certificates if `root_certificates` is None. transport : Optional[str] (default: None) Configure the transport layer. Allowed values: - 'grpc-bidi': gRPC, bidirectional streaming @@ -301,16 +355,25 @@ def start_numpy_client( Examples -------- - Starting a client with an insecure server connection: + Starting a gRPC client with an insecure server connection: + + >>> start_numpy_client( + >>> server_address=localhost:8080, + >>> client=FlowerClient(), + >>> ) + + Starting an SSL-enabled gRPC client using system certificates: >>> start_numpy_client( >>> server_address=localhost:8080, >>> client=FlowerClient(), + >>> insecure=False, >>> ) - Starting an SSL-enabled gRPC client: + Starting an SSL-enabled gRPC client using provided certificates: >>> from pathlib import Path + >>> >>> start_numpy_client( >>> server_address=localhost:8080, >>> client=FlowerClient(), @@ -340,6 +403,7 @@ def start_numpy_client( client=wrp_client, grpc_max_message_length=grpc_max_message_length, root_certificates=root_certificates, + insecure=insecure, transport=transport, ) @@ -348,7 +412,7 @@ def _init_connection( transport: Optional[str], server_address: str ) -> Tuple[ Callable[ - [str, int, Union[bytes, str, None]], + [str, bool, int, Union[bytes, str, None]], ContextManager[ Tuple[ Callable[[], Optional[TaskIns]], diff --git a/src/py/flwr/client/grpc_client/connection.py b/src/py/flwr/client/grpc_client/connection.py index cbef4ef99051..335d28e72828 100644 --- a/src/py/flwr/client/grpc_client/connection.py +++ b/src/py/flwr/client/grpc_client/connection.py @@ -45,6 +45,7 @@ def on_channel_state_change(channel_connectivity: str) -> None: @contextmanager def grpc_connection( server_address: str, + insecure: bool, max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, root_certificates: Optional[Union[bytes, str]] = None, ) -> Iterator[ @@ -100,6 +101,7 @@ def grpc_connection( channel = create_channel( server_address=server_address, + insecure=insecure, root_certificates=root_certificates, max_message_length=max_message_length, ) diff --git a/src/py/flwr/client/grpc_client/connection_test.py b/src/py/flwr/client/grpc_client/connection_test.py index 0485fa41db35..e5944230e5af 100644 --- a/src/py/flwr/client/grpc_client/connection_test.py +++ b/src/py/flwr/client/grpc_client/connection_test.py @@ -93,7 +93,7 @@ def test_integration_connection() -> None: def run_client() -> int: messages_received: int = 0 - with grpc_connection(server_address=f"[::]:{port}") as conn: + with grpc_connection(server_address=f"[::]:{port}", insecure=True) as conn: receive, send, _, _ = conn # Setup processing loop diff --git a/src/py/flwr/client/grpc_rere_client/connection.py b/src/py/flwr/client/grpc_rere_client/connection.py index 424e413dc484..30d407a52c53 100644 --- a/src/py/flwr/client/grpc_rere_client/connection.py +++ b/src/py/flwr/client/grpc_rere_client/connection.py @@ -51,10 +51,9 @@ def on_channel_state_change(channel_connectivity: str) -> None: @contextmanager def grpc_request_response( server_address: str, + insecure: bool, max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, # pylint: disable=W0613 - root_certificates: Optional[ - Union[bytes, str] - ] = None, # pylint: disable=unused-argument + root_certificates: Optional[Union[bytes, str]] = None, ) -> Iterator[ Tuple[ Callable[[], Optional[TaskIns]], @@ -95,6 +94,7 @@ def grpc_request_response( channel = create_channel( server_address=server_address, + insecure=insecure, root_certificates=root_certificates, max_message_length=max_message_length, ) diff --git a/src/py/flwr/client/rest_client/connection.py b/src/py/flwr/client/rest_client/connection.py index 092e543bf55b..d22b246dbd61 100644 --- a/src/py/flwr/client/rest_client/connection.py +++ b/src/py/flwr/client/rest_client/connection.py @@ -61,6 +61,7 @@ # pylint: disable-next=too-many-statements def http_request_response( server_address: str, + insecure: bool, # pylint: disable=unused-argument max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, # pylint: disable=W0613 root_certificates: Optional[ Union[bytes, str] diff --git a/src/py/flwr/common/grpc.py b/src/py/flwr/common/grpc.py index 2857048f62a0..9d0543ea8c75 100644 --- a/src/py/flwr/common/grpc.py +++ b/src/py/flwr/common/grpc.py @@ -27,10 +27,19 @@ def create_channel( server_address: str, + insecure: bool, root_certificates: Optional[bytes] = None, max_message_length: int = GRPC_MAX_MESSAGE_LENGTH, ) -> grpc.Channel: """Create a gRPC channel, either secure or insecure.""" + # Check for conflicting parameters + if insecure and root_certificates is not None: + raise ValueError( + "Invalid configuration: 'root_certificates' should not be provided " + "when 'insecure' is set to True. For an insecure connection, omit " + "'root_certificates', or set 'insecure' to False for a secure connection." + ) + # Possible options: # https://github.com/grpc/grpc/blob/v1.43.x/include/grpc/impl/codegen/grpc_types.h channel_options = [ @@ -38,14 +47,14 @@ def create_channel( ("grpc.max_receive_message_length", max_message_length), ] - if root_certificates is not None: + if insecure: + channel = grpc.insecure_channel(server_address, options=channel_options) + log(INFO, "Opened insecure gRPC connection (no certificates were passed)") + else: ssl_channel_credentials = grpc.ssl_channel_credentials(root_certificates) channel = grpc.secure_channel( server_address, ssl_channel_credentials, options=channel_options ) log(INFO, "Opened secure gRPC connection using certificates") - else: - channel = grpc.insecure_channel(server_address, options=channel_options) - log(INFO, "Opened insecure gRPC connection (no certificates were passed)") return channel diff --git a/src/py/flwr/driver/grpc_driver.py b/src/py/flwr/driver/grpc_driver.py index a25de6f9f666..7dd0a0f501c5 100644 --- a/src/py/flwr/driver/grpc_driver.py +++ b/src/py/flwr/driver/grpc_driver.py @@ -66,6 +66,7 @@ def connect(self) -> None: return self.channel = create_channel( server_address=self.driver_service_address, + insecure=(self.certificates is None), root_certificates=self.certificates, ) self.stub = DriverStub(self.channel) From 1f33ba84da35b08b8b9ce53a4179b51ed0a377a4 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 19:20:43 +0100 Subject: [PATCH 5/9] Add HTTPS changes to changelog (#2649) Co-authored-by: Daniel J. Beutel --- doc/source/ref-changelog.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index 9aa169dd97bd..b1282d898586 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -24,14 +24,20 @@ We would like to give our special thanks to all the contributors who made the ne - **Update REST API to support create and delete nodes** ([#2283](https://github.com/adap/flower/pull/2283)) -- **Update the Android SDK** ([#2187](https://github/com/adap/flower/pull/2187)) +- **Update the Android SDK** ([#2187](https://github.com/adap/flower/pull/2187)) Add gRPC request-response capability to the Android SDK. -- **Update the C++ SDK** ([#2537](https://github/com/adap/flower/pull/2537), [#2528](https://github/com/adap/flower/pull/2528), [#2523](https://github.com/adap/flower/pull/2523), [#2522](https://github.com/adap/flower/pull/2522)) +- **Update the C++ SDK** ([#2537](https://github.com/adap/flower/pull/2537), [#2528](https://github.com/adap/flower/pull/2528), [#2523](https://github.com/adap/flower/pull/2523), [#2522](https://github.com/adap/flower/pull/2522)) Add gRPC request-response capability to the C++ SDK. +- **Make HTTPS the new default** ([#2591](https://github.com/adap/flower/pull/2591), [#2636](https://github.com/adap/flower/pull/2636)) + + Flower is moving to HTTPS by default. The new `flower-server` requires passing `--certificates`, but users can enable `--insecure` to use HTTP for prototyping. The same applies to `flower-client`, which can either use user-provided credentials or gRPC-bundled certificates to connect to an HTTPS-enabled server or requires opt-out via passing `--insecure` to enable insecure HTTP connections. + + For backward compatibility, `start_client()` and `start_numpy_client()` will still start in insecure mode by default. In a future release, insecure connections will require user opt-in by passing `insecure=True`. + - **Unify client API** ([#2303](https://github.com/adap/flower/pull/2303), [#2390](https://github.com/adap/flower/pull/2390), [#2493](https://github.com/adap/flower/pull/2493)) Using the `client_fn`, Flower clients can interchangeably run as standalone processes (i.e. via `start_client`) or in simulation (i.e. via `start_simulation`) without requiring changes to how the client class is defined and instantiated. The `to_client()` function is introduced to convert a `NumPyClient` to a `Client`. From bcf2dfc2dd890362ada6781d914c946eaa58df76 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 19:35:05 +0100 Subject: [PATCH 6/9] Use version instead of commit hash for action (#2650) --- .github/workflows/framework-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/framework-release.yml b/.github/workflows/framework-release.yml index 2d2b7d8a4c4f..1c55894a127b 100644 --- a/.github/workflows/framework-release.yml +++ b/.github/workflows/framework-release.yml @@ -53,7 +53,7 @@ jobs: cat body.md - name: Release - uses: softprops/action-gh-release@de2c0eb + uses: softprops/action-gh-release@v1 with: body_path: ./body.md draft: true From 231a81f950cfe8f0c45d16ad1b278fa779888909 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 20:06:44 +0100 Subject: [PATCH 7/9] Update version to 1.7.0 (#2651) --- baselines/doc/source/conf.py | 2 +- doc/source/conf.py | 2 +- .../contributor-how-to-install-development-versions.rst | 4 ++-- examples/doc/source/conf.py | 2 +- pyproject.toml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/baselines/doc/source/conf.py b/baselines/doc/source/conf.py index dd43080f299e..e1055aaed216 100644 --- a/baselines/doc/source/conf.py +++ b/baselines/doc/source/conf.py @@ -36,7 +36,7 @@ author = "The Flower Authors" # The full version, including alpha/beta/rc tags -release = "1.6.0" +release = "1.7.0" # -- General configuration --------------------------------------------------- diff --git a/doc/source/conf.py b/doc/source/conf.py index 87860e2c6e7b..8077d26aa6ae 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -85,7 +85,7 @@ author = "The Flower Authors" # The full version, including alpha/beta/rc tags -release = "1.6.0" +release = "1.7.0" # -- General configuration --------------------------------------------------- diff --git a/doc/source/contributor-how-to-install-development-versions.rst b/doc/source/contributor-how-to-install-development-versions.rst index e8028a35d706..243f4ef97e8e 100644 --- a/doc/source/contributor-how-to-install-development-versions.rst +++ b/doc/source/contributor-how-to-install-development-versions.rst @@ -59,5 +59,5 @@ Open a development version of the same notebook from branch `branch-name` by cha Install a `whl` on Google Colab: 1. In the vertical icon grid on the left hand side, select ``Files`` > ``Upload to session storage`` -2. Upload the whl (e.g., ``flwr-1.6.0-py3-none-any.whl``) -3. Change ``!pip install -q 'flwr[simulation]' torch torchvision matplotlib`` to ``!pip install -q 'flwr-1.6.0-py3-none-any.whl[simulation]' torch torchvision matplotlib`` +2. Upload the whl (e.g., ``flwr-1.7.0-py3-none-any.whl``) +3. Change ``!pip install -q 'flwr[simulation]' torch torchvision matplotlib`` to ``!pip install -q 'flwr-1.7.0-py3-none-any.whl[simulation]' torch torchvision matplotlib`` diff --git a/examples/doc/source/conf.py b/examples/doc/source/conf.py index dcb1788ea336..01cbb48c1587 100644 --- a/examples/doc/source/conf.py +++ b/examples/doc/source/conf.py @@ -27,7 +27,7 @@ author = "The Flower Authors" # The full version, including alpha/beta/rc tags -release = "1.6.0" +release = "1.7.0" # -- General configuration --------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index ff71298c1c2d..2349d554a409 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "flwr" -version = "1.6.0" +version = "1.7.0" description = "Flower: A Friendly Federated Learning Framework" license = "Apache-2.0" authors = ["The Flower Authors "] From b3c530a8ae694e2633e4bc243347a83c73ff3253 Mon Sep 17 00:00:00 2001 From: Charles Beauville Date: Tue, 28 Nov 2023 20:18:34 +0100 Subject: [PATCH 8/9] Add unreleased to changelog (#2652) --- doc/source/ref-changelog.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index b1282d898586..0cf7aca63941 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -1,5 +1,7 @@ # Changelog +## Unreleased + ## v1.6.0 (2023-11-28) ### Thanks to our contributors From 02bb3ba734b85c5fcb7bca6d9a7381562f3fd8ce Mon Sep 17 00:00:00 2001 From: Javier Date: Wed, 29 Nov 2023 11:54:09 +0000 Subject: [PATCH 9/9] Fix vertical-fl example (#2655) Co-authored-by: Charles Beauville --- examples/vertical-fl/pyproject.toml | 6 +++--- examples/vertical-fl/requirements.txt | 2 +- examples/vertical-fl/simulation.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/vertical-fl/pyproject.toml b/examples/vertical-fl/pyproject.toml index 3c3307f24f08..14771c70062f 100644 --- a/examples/vertical-fl/pyproject.toml +++ b/examples/vertical-fl/pyproject.toml @@ -3,14 +3,14 @@ requires = ["poetry-core>=1.4.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] -name = "quickstart-pytorch" +name = "vertical-fl" version = "0.1.0" -description = "PyTorch Federated Learning Quickstart with Flower" +description = "PyTorch Vertical FL with Flower" authors = ["The Flower Authors "] [tool.poetry.dependencies] python = ">=3.8,<3.11" -flwr = ">=1.0,<2.0" +flwr = { extras = ["simulation"], version = ">=1.0,<2.0" } torch = "2.1.0" matplotlib = "3.7.3" scikit-learn = "1.3.2" diff --git a/examples/vertical-fl/requirements.txt b/examples/vertical-fl/requirements.txt index f5c4f6f7b453..aee341e4c554 100644 --- a/examples/vertical-fl/requirements.txt +++ b/examples/vertical-fl/requirements.txt @@ -1,4 +1,4 @@ -flwr>=1.0, <2.0 +flwr[simulation]>=1.0, <2.0 torch==2.1.0 matplotlib==3.7.3 scikit-learn==1.3.2 diff --git a/examples/vertical-fl/simulation.py b/examples/vertical-fl/simulation.py index a6b1184dd76c..095ec6e0c7ea 100644 --- a/examples/vertical-fl/simulation.py +++ b/examples/vertical-fl/simulation.py @@ -2,6 +2,7 @@ import numpy as np from strategy import Strategy from client import FlowerClient +from pathlib import Path from task import get_partitions_and_label partitions, label = get_partitions_and_label() @@ -19,4 +20,6 @@ def client_fn(cid): strategy=Strategy(label), ) -np.save("_static/results/hist.npy", hist) +results_dir = Path("_static/results") +results_dir.mkdir(exist_ok=True) +np.save(str(results_dir/"hist.npy"), hist)