From 306e9f63542c2c74575932664ceff39f803f75e4 Mon Sep 17 00:00:00 2001 From: Aml Hassan Esmil Date: Thu, 7 Dec 2023 13:38:41 +0200 Subject: [PATCH] Add FL+XGBoost Baseline (#2226) Co-authored-by: jafermarq --- baselines/hfedxgboost/.gitignore | 2 + baselines/hfedxgboost/LICENSE | 202 +++++++ baselines/hfedxgboost/README.md | 297 +++++++++ baselines/hfedxgboost/hfedxgboost/__init__.py | 1 + baselines/hfedxgboost/hfedxgboost/client.py | 294 +++++++++ .../conf/Centralized_Baseline.yaml | 22 + .../hfedxgboost/hfedxgboost/conf/base.yaml | 55 ++ ...zed_basline_all_datasets_paper_config.yaml | 46 ++ .../clients/YearPredictionMSD_10_clients.yaml | 9 + .../clients/YearPredictionMSD_2_clients.yaml | 9 + .../clients/YearPredictionMSD_5_clients.yaml | 9 + .../conf/clients/a9a_10_clients.yaml | 9 + .../conf/clients/a9a_2_clients.yaml | 9 + .../conf/clients/a9a_5_clients.yaml | 9 + .../conf/clients/abalone_10_clients.yaml | 9 + .../conf/clients/abalone_2_clients.yaml | 9 + .../conf/clients/abalone_5_clients.yaml | 9 + .../conf/clients/cod_rna_10_clients.yaml | 9 + .../conf/clients/cod_rna_2_clients.yaml | 9 + .../conf/clients/cod_rna_5_clients.yaml | 9 + .../conf/clients/cpusmall_10_clients.yaml | 9 + .../conf/clients/cpusmall_2_clients.yaml | 9 + .../conf/clients/cpusmall_5_clients.yaml | 9 + .../conf/clients/ijcnn1_10_clients.yaml | 9 + .../conf/clients/ijcnn1_2_clients.yaml | 9 + .../conf/clients/ijcnn1_5_clients.yaml | 9 + .../conf/clients/paper_10_clients.yaml | 9 + .../conf/clients/paper_2_clients.yaml | 9 + .../conf/clients/paper_5_clients.yaml | 9 + .../conf/clients/space_ga_10_clients.yaml | 9 + .../conf/clients/space_ga_2_clients.yaml | 9 + .../conf/clients/space_ga_5_clients.yaml | 9 + .../conf/dataset/YearPredictionMSD.yaml | 5 + .../hfedxgboost/conf/dataset/a9a.yaml | 5 + .../hfedxgboost/conf/dataset/abalone.yaml | 5 + .../hfedxgboost/conf/dataset/cod_rna.yaml | 5 + .../hfedxgboost/conf/dataset/cpusmall.yaml | 5 + .../hfedxgboost/conf/dataset/ijcnn1.yaml | 5 + .../hfedxgboost/conf/dataset/space_ga.yaml | 5 + .../dataset/task/Binary_Classification.yaml | 14 + .../conf/dataset/task/Regression.yaml | 14 + .../hfedxgboost/conf/wandb/default.yaml | 6 + ...YearPredictionMSD_xgboost_centralized.yaml | 11 + .../abalone_xgboost_centralized.yaml | 11 + .../cpusmall_xgboost_centralized.yaml | 11 + .../paper_xgboost_centralized.yaml | 11 + baselines/hfedxgboost/hfedxgboost/dataset.py | 142 +++++ .../hfedxgboost/dataset_preparation.py | 262 ++++++++ baselines/hfedxgboost/hfedxgboost/main.py | 143 +++++ baselines/hfedxgboost/hfedxgboost/models.py | 130 ++++ baselines/hfedxgboost/hfedxgboost/server.py | 406 +++++++++++++ baselines/hfedxgboost/hfedxgboost/strategy.py | 5 + baselines/hfedxgboost/hfedxgboost/sweep.yaml | 28 + baselines/hfedxgboost/hfedxgboost/utils.py | 566 ++++++++++++++++++ baselines/hfedxgboost/pyproject.toml | 145 +++++ baselines/hfedxgboost/results.csv | 155 +++++ baselines/hfedxgboost/results_centralized.csv | 2 + doc/source/ref-changelog.md | 4 + 58 files changed, 3236 insertions(+) create mode 100644 baselines/hfedxgboost/.gitignore create mode 100644 baselines/hfedxgboost/LICENSE create mode 100644 baselines/hfedxgboost/README.md create mode 100644 baselines/hfedxgboost/hfedxgboost/__init__.py create mode 100644 baselines/hfedxgboost/hfedxgboost/client.py create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/Centralized_Baseline.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/base.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/centralized_basline_all_datasets_paper_config.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/paper_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/paper_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/paper_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_10_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_2_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_5_clients.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/YearPredictionMSD.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/a9a.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/abalone.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/cod_rna.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/cpusmall.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/ijcnn1.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/space_ga.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Binary_Classification.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Regression.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/wandb/default.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/YearPredictionMSD_xgboost_centralized.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/abalone_xgboost_centralized.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/cpusmall_xgboost_centralized.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/paper_xgboost_centralized.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/dataset.py create mode 100644 baselines/hfedxgboost/hfedxgboost/dataset_preparation.py create mode 100644 baselines/hfedxgboost/hfedxgboost/main.py create mode 100644 baselines/hfedxgboost/hfedxgboost/models.py create mode 100644 baselines/hfedxgboost/hfedxgboost/server.py create mode 100644 baselines/hfedxgboost/hfedxgboost/strategy.py create mode 100644 baselines/hfedxgboost/hfedxgboost/sweep.yaml create mode 100644 baselines/hfedxgboost/hfedxgboost/utils.py create mode 100644 baselines/hfedxgboost/pyproject.toml create mode 100644 baselines/hfedxgboost/results.csv create mode 100644 baselines/hfedxgboost/results_centralized.csv diff --git a/baselines/hfedxgboost/.gitignore b/baselines/hfedxgboost/.gitignore new file mode 100644 index 000000000000..3d66a02ea3cc --- /dev/null +++ b/baselines/hfedxgboost/.gitignore @@ -0,0 +1,2 @@ +dataset/ +outputs/ diff --git a/baselines/hfedxgboost/LICENSE b/baselines/hfedxgboost/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/baselines/hfedxgboost/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/baselines/hfedxgboost/README.md b/baselines/hfedxgboost/README.md new file mode 100644 index 000000000000..29702496370b --- /dev/null +++ b/baselines/hfedxgboost/README.md @@ -0,0 +1,297 @@ +--- +title: Gradient-less Federated Gradient Boosting Trees with Learnable Learning Rates +URL: https://arxiv.org/abs/2304.07537 +labels: [cross-silo, tree-based, XGBoost, Classification, Regression, Tabular] +dataset: [a9a, cod-rna, ijcnn1, space_ga, cpusmall, YearPredictionMSD] +--- + +# Gradient-less Federated Gradient Boosting Trees with Learnable Learning Rates + +> Note: If you use this baseline in your work, please remember to cite the original authors of the paper as well as the Flower paper. + +**Paper:** [arxiv.org/abs/2304.07537](https://arxiv.org/abs/2304.07537) + +**Authors:** Chenyang Ma, Xinchi Qiu, Daniel J. Beutel, Nicholas D. Laneearly_stop_patience_rounds: 100 + +**Abstract:** The privacy-sensitive nature of decentralized datasets and the robustness of eXtreme Gradient Boosting (XGBoost) on tabular data raise the need to train XGBoost in the context of federated learning (FL). Existing works on federated XGBoost in the horizontal setting rely on the sharing of gradients, which induce per-node level communication frequency and serious privacy concerns. To alleviate these problems, we develop an innovative framework for horizontal federated XGBoost which does not depend on the sharing of gradients and simultaneously boosts privacy and communication efficiency by making the learning rates of the aggregated tree ensembles are learnable. We conduct extensive evaluations on various classification and regression datasets, showing our approach achieve performance comparable to the state-of-the-art method and effectively improves communication efficiency by lowering both communication rounds and communication overhead by factors ranging from 25x to 700x. + + +## About this baseline + +**What’s implemented:** The code in this directory replicates the experiments in "Gradient-less Federated Gradient Boosting Trees with Learnable Learning Rates" (Ma et al., 2023) for a9a, cod-rna, ijcnn1, space_ga datasets, which proposed the FedXGBllr algorithm. Concretely, it replicates the results for a9a, cod-rna, ijcnn1, space_ga datasets in Table 2. + +**Datasets:** a9a, cod-rna, ijcnn1, space_ga + +**Hardware Setup:** Most of the experiments were done on a machine with an Intel® Core™ i7-6820HQ Processor, that processor got 4 cores and 8 threads. + +**Contributors:** [Aml Hassan Esmil](https://github.com/Aml-Hassan-Abd-El-hamid) + +## Experimental Setup + +**Task:** Tabular classification and regression + +**Model:** XGBoost model combined with 1-layer CNN + +**Dataset:** +This baseline only includes 7 datasets with a focus on 4 of them (a9a, cod-rna, ijcnn1, space_ga). + +Each dataset can be partitioned across 2, 5 or 10 clients in an IID distribution. + +| task type | Dataset | no.of features | no.of samples | +| :---: | :---: | :---: | :---: | +| Binary classification | a9a
cod-rna
ijcnn1 | 123
8
22 | 32,561
59,5358
49,990 | +| Regression | abalone
cpusmall
space_ga
YearPredictionMSD | 8
12
6
90 | 4,177
8,192
3,167
515,345 | + + +**Training Hyperparameters:** +For the centralized model, the paper's hyperparameters were mostly used as they give very good results -except for abalone and cpusmall-, here are the used hyperparameters -they can all be found in the `yaml` file named `paper_xgboost_centralized`: + +| Hyperparameter name | value | +| -- | -- | +| n_estimators | 500 | +| max_depth | 8 | +| subsample | 0.8 | +| learning_rate | .1 | +| colsample_bylevel | 1 | +| colsample_bynode | 1 | +| colsample_bytree | 1 | +| alpha | 5 | +| gamma | 5 | +| num_parallel_tree | 1 | +| min_child_weight | 1 | + +Here are all the original hyperparameters for the federated horizontal XGBoost model -hyperparameters that are used only in the XGBoost model are initialized with xgb same for the ones only used in Adam-: + +| Hyperparameter name | value | +| -- | -- | +| n_estimators | 500/no.of clients | +| xgb max_depth | 8 | +| xgb subsample | 0.8 | +| xgb learning_rate | .1 | +| xgb colsample_bylevel | 1 | +| xgb colsample_bynode | 1 | +| xgb colsample_bytree | 1 | +| xgb alpha | 5 | +| xgb gamma | 5 | +| xgb num_parallel_tree | 1 | +| xgb min_child_weight | 1 | +| Adam learning rate | .0001 | +| Adam Betas | 0.5, 0.999 | +| no.of iterations for the CNN model | 100 | + +Those hyperparameters did well for most datasets but for some datasets, it wasn't giving the best performance so a fine-tuning journey has started in order to achieve better results.
+At first, it was a manual process basically experiencing different values for some groups of hyperparameters to explore those hyperparameters's effect on the performance of different datasets until I decided to focus on those groups of the following hyperparameters as they seemed to have the major effect on different datasets performances: +| Hyperparameter name | +| -- | +| n_estimators | +| xgb max_depth | +| Adam learning rate | +| no.of iterations for the CNN model | + +All the final new values for those hyperparameters can be found in 3 `yaml` files named `dataset_name__clients` and all the original values for those hyperparameters can be found in 3 `yaml` files named `paper__clients`. This resulted in a large number of config files 3*7+3= 24 config files in the `clients` folder. + +## Environment Setup + +These steps assume you have already installed `Poetry` and `pyenv`. In the this directory (i.e. `/baselines/hfedxgboost`) where you can see `pyproject.toml`, execute the following commands in your terminal: + +```bash +# Set python version +pyenv local 3.10.6 +# Tell Poetry to use it +poetry env use 3.10.6 +# Install all dependencies +poetry install +# Activate your environment +poetry shell +``` + +## Running the Experiments + +With your environment activated you can run the experiments directly. The datasets will be downloaded automatically. + +```bash +# to run the experiments for the centralized model with customized hyperparameters run +python -m hfedxgboost.main --config-name Centralized_Baseline dataset= xgboost_params_centralized= +#e.g +# to run the centralized model with customized hyperparameters for cpusmall dataset +python -m hfedxgboost.main --config-name Centralized_Baseline dataset=cpusmall xgboost_params_centralized=cpusmall_xgboost_centralized + +# to run the federated version for any dataset with no.of clients +python -m hfedxgboost.main dataset= clients= +# for example +# to run the federated version for a9a dataset with 5 clients +python -m hfedxgboost.main dataset=a9a clients=a9a_5_clients + +# if you wish to change any parameters from any config file from the terminal, then you should follow this formula +python -m hfedxgboost.main folder=config_file_name folder.parameter_name=its_new_value +#e.g: +python -m hfedxgboost.main --config-name Centralized_Baseline dataset=abalone xgboost_params_centralized=abalone_xgboost_centralized xgboost_params_centralized.max_depth=8 dataset.train_ratio=.80 +``` + + +## Expected Results + +This section shows how to reproduce some of the results in the paper. Tables 2 and 3 were obtained using different hyperparameters than those indicated in the paper. Without these some experimetn exhibited worse performance. Still, some results remain far from those in the original paper. + +### Table 1: Centralized Evaluation +```bash +# to run all the experiments for the centralized model with the original paper config for all the datasets +# gives the output shown in Table 1 +python -m hfedxgboost.main --config-name centralized_basline_all_datasets_paper_config + +# Please note that unlike in the federated experiments, the results will be only printed on the terminal +# and won't be logged into a file. +``` +| Dataset | task type | test result | +| :---: | :---: | :---: | +| a9a | Binary classification | 84.9% | +| cod-rna | Binary classification | 97.3% | +| ijcnn1 | Binary classification | 98.7% | +| abalone | Regression | 4.6 | +| cpusmall | Regression | 9 | +| space_ga | Regression | .032 | +| YearPredictionMSD | Regression | 76.41 | + +### Table 2: Federated Binary Classification + +```bash +# Results for a9a dataset in table 2 +python -m hfedxgboost.main --multirun clients=a9a_2_clients,a9a_5_clients,a9a_10_clients dataset=a9a + +# Results for cod_rna dataset in table 2 +python -m hfedxgboost.main --multirun clients=cod_rna_2_clients,cod_rna_5_clients,cod_rna_10_clients dataset=cod_rna + +# Results for ijcnn1 dataset in table 2 +python -m hfedxgboost.main --multirun clients=ijcnn1_2_clients,ijcnn1_5_clients,ijcnn1_10_clients dataset=ijcnn1 +``` + +| Dataset | task type |no. of clients | server-side test Accuracy | +| :---: | :---: | :---: | :---: | +| a9a | Binary Classification | 2
5
10 | 84.4%
84.2%
83.7% | +| cod_rna | Binary Classification | 2
5
10 | 96.4%
96.2%
95.0% | +| ijcnn1 | Binary Classification |2
5
10 | 98.0%
97.28%
96.8% | + + +### Table 3: Federated Regression +```bash +# Notice that: the MSE results shown in the tables usually happen in early FL rounds (instead in the last round/s) +# Results for space_ga dataset in table 3 +python -m hfedxgboost.main --multirun clients=space_ga_2_clients,space_ga_5_clients,space_ga_10_clients dataset=space_ga + +# Results for abalone dataset in table 3 +python -m hfedxgboost.main --multirun clients=abalone_2_clients,abalone_5_clients,abalone_10_clients dataset=abalone + +# Results for cpusmall dataset in table 3 +python -m hfedxgboost.main --multirun clients=cpusmall_2_clients,cpusmall_5_clients,cpusmall_10_clients dataset=cpusmall + +# Results for YearPredictionMSD_2 dataset in table 3 +python -m hfedxgboost.main --multirun clients=YearPredictionMSD_2_clients,YearPredictionMSD_5_clients,YearPredictionMSD_10_clients dataset=YearPredictionMSD +``` + +| Dataset | task type |no. of clients | server-side test MSE | +| :---: | :---: | :---: | :---: | +| space_ga | Regression | 2
5
10 | 0.024
0.033
0.034 | +| abalone | Regression | 2
5
10 | 5.5
6.87
7.5 | +| cpusmall | Regression | 2
5
10 | 13
15.13
15.28 | +| YearPredictionMSD | Regression | 2
5
10 | 119
118
118 | + + +## Doing your own finetuning + +There are 3 main things that you should consider: + +1- You can use WandB to automate the fine-tuning process, modify the `sweep.yaml` file to control your experiments settings including your search methods, values to choose from, etc. Below we demonstrate how to run the `wandb` sweep. +If you're new to `wandb` you might want to read the following resources to [do hyperparameter tuning with W&B+PyTorch](https://colab.research.google.com/github/wandb/examples/blob/master/colabs/pytorch/Organizing_Hyperparameter_Sweeps_in_PyTorch_with_W%26B.ipynb), and [use W&B alongside Hydra](https://wandb.ai/adrishd/hydra-example/reports/Configuring-W-B-Projects-with-Hydra--VmlldzoxNTA2MzQw). + +``` +# Remember to activate the poetry shell +poetry shell + +# login to your wandb account +wandb login + +# Inside the folder flower/baselines/hfedxgboost/hfedxgboost run the commands below + +# Initiate WandB sweep +wandb sweep sweep.yaml + +# that command -if ran with no error- will return a line that contains +# the command that you can use to run the sweep agent, it'll look something like that: + +wandb agent /flower-baselines_hfedxgboost_hfedxgboost/ + +``` + +2- The config files named `__clients.yaml` are meant to keep the final hyperparameters values, so whenever you think you're done with fine-tuning some hyperparameters, add them to their config files so the one after you can use them. + +3- To help with the fine-tuning of the hyperparameters process, there are 2 classes in the utils.py that write down the used hyperparameters in the experiments and the results for that experiment in 2 separate CSV files, some of the hyperparameters used in the experiments done during building this baseline can be found in results.csv and results_centralized.csv files.
+More important, those 2 classes focus on writing down only the hyperparameters that I thought was important so if you're interested in experimenting with other hyperparameters, don't forget to add them to the writers classes so you can track them more easily, especially if you intend to do some experiments away from WandB. + + +## How to add a new dataset + +This code doesn't cover all the datasets from the paper yet, so if you wish to add a new dataset, here are the steps: + +**1- you need to download the dataset from its source:** +- In the `dataset_preparation.py` file, specifically in the `download_data` function add the code to download your dataset -or if you already downloaded it manually add the code to return its file path- it could look something like the following example: +``` +if dataset_name=="": + DATASET_PATH=os.path.join(ALL_DATASETS_PATH, "") + if not os.path.exists(DATASET_PATH): + os.makedirs(DATASET_PATH) + urllib.request.urlretrieve( + "", + f"{os.path.join(DATASET_PATH, '')}", + ) + urllib.request.urlretrieve( + "", + f"{os.path.join(DATASET_PATH, '')}", + ) + # if the 2 files of your dataset are divided into training and test file put the training then test ✅ + return [os.path.join(DATASET_PATH, ''),os.path.join(DATASET_PATH, '')] +``` +that function will be called in the `dataset.py` file in the `load_single_dataset` function and the different files of your dataset will be concatenated -if your dataset is one file then nothing will happen it will just be loaded- using the `datafiles_fusion` function from the `dataset_preparation.py` file. + +:warning: if any of your dataset's files end with `.bz2` you have to add the following piece of code before the return line and inside the `if` condition +``` +for filepath in os.listdir(DATASET_PATH): + abs_filepath = os.path.join(DATASET_PATH, filepath) + with bz2.BZ2File(abs_filepath) as fr, open(abs_filepath[:-4], "wb") as fw: + shutil.copyfileobj(fr, fw) +``` + +:warning: `datafiles_fusion` function uses `sklearn.datasets.load_svmlight_file` to load the dataset, if your dataset is `csv` or something that function won't work on it and you will have to alter the `datafiles_fusion` function to work with you dataset files format. + +**2- Add config files for your dataset:** + +**a- config files for the centralized baseline:** + +- To run the centralized model on your dataset with the original hyper-parameters from the paper alongside all the other datasets added before just do the following step: + - in the dictionary called `dataset_tasks` in the `utils.py` file add your dataset name as a key -the same name that you put in the `download_data` function in the step before- and add its task type, this code performs for 2 tasks: `BINARY` which is binary classification or `REG` which is regression. + +- To run the centralized model on your dataset you need to create a config file `.yaml` in the `xgboost_params_centralized` folder and another .yaml file in the `dataset` folder -you will find that one of course inside the `conf` folder :) - and you need to specify the hyper-parameters of your choice for the xgboost model + + - the .yaml file in the `dataset` folder should look something like this: + ``` + defaults: + - task: + dataset_name: "" + train_ratio: + early_stop_patience_rounds: + ``` + - the .yaml file in the `xgboost_params_centralized` folder should contain the values for all the hyper-parameters of your choice for the xgboost model + +You can skip this whole step and use the paper default hyper-parameters from the paper, they're all written in the "paper__clients.yaml" files.
+**b- config files for the federated baseline:** + +To run the federated baseline with your dataset using your customized hyper-parameters, you need first to create the .yaml file in the `dataset` folder that was mentioned before and you need to create config files that contain the no.of the clients and it should look something like this: +``` +n_estimators_client: +num_rounds: +client_num: +num_iterations: +xgb: + max_depth: +CNN: + lr: +``` diff --git a/baselines/hfedxgboost/hfedxgboost/__init__.py b/baselines/hfedxgboost/hfedxgboost/__init__.py new file mode 100644 index 000000000000..543147a05591 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/__init__.py @@ -0,0 +1 @@ +"""hfedxgboost baseline package.""" diff --git a/baselines/hfedxgboost/hfedxgboost/client.py b/baselines/hfedxgboost/hfedxgboost/client.py new file mode 100644 index 000000000000..22435e20415b --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/client.py @@ -0,0 +1,294 @@ +"""Define your client class and a function to construct such clients. + +Please overwrite `flwr.client.NumPyClient` or `flwr.client.Client` and create a function +to instantiate your client. +""" +from typing import Any, Tuple + +import flwr as fl +import torch +from flwr.common import ( + Code, + EvaluateIns, + EvaluateRes, + FitIns, + FitRes, + GetParametersRes, + Status, + ndarrays_to_parameters, + parameters_to_ndarrays, +) +from hydra.utils import instantiate +from omegaconf import DictConfig +from torch.utils.data import DataLoader + +from hfedxgboost.models import CNN, fit_xgboost +from hfedxgboost.utils import single_tree_preds_from_each_client + + +class FlClient(fl.client.Client): + """Custom class contains the methods that the client need.""" + + def __init__( + self, + cfg: DictConfig, + trainloader: DataLoader, + valloader: DataLoader, + cid: str, + ): + self.cid = cid + self.config = cfg + + self.trainloader_original = trainloader + self.valloader_original = valloader + self.valloader: Any + + # instantiate model + self.net = CNN(cfg) + + # determine device + self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + def train_one_loop(self, data, optimizer, metric_fn, criterion): + """Trains the neural network model for one loop iteration. + + Parameters + ---------- + data (tuple): A tuple containing the inputs and + labels for the training data, where the input represent the predictions + of the trees from the tree ensemples of the clients. + + Returns + ------- + loss (float): The value of the loss function after the iteration. + metric_val * n_samples (float): The value of the chosen evaluation metric + (accuracy or MSE) after the iteration. + n_samples (int): The number of samples used for training in the iteration. + """ + tree_outputs, labels = data[0].to(self.device), data[1].to(self.device) + optimizer.zero_grad() + + outputs = self.net(tree_outputs) + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # Collected training loss and accuracy statistics + n_samples = labels.size(0) + metric_val = metric_fn(outputs, labels.type(torch.int)) + + return loss.item(), metric_val * n_samples, n_samples + + def train( + self, + net: CNN, + trainloader: DataLoader, + num_iterations: int, + ) -> Tuple[float, float, int]: + """Train CNN model on a given dataset(trainloader) for(num_iterations). + + Parameters + ---------- + net (CNN): The convolutional neural network to be trained. + trainloader (DataLoader): The data loader object containing + the training dataset. + num_iterations (int): The number of iterations or batches to + be processed by the network. + + Returns + ------- + Tuple[float, float, int]: A tuple containing the average loss per sample, + the average evaluation result per sample, and the total number of training + samples processed. + + Note: + + - The training is formulated in terms of the number of updates or iterations + processed by the network. + """ + net.train() + total_loss, total_result, total_n_samples = 0.0, 0.0, 0 + + # Unusually, this training is formulated in terms of number of + # updates/iterations/batches processed + # by the network. This will be helpful later on, when partitioning the + # data across clients: resulting + # in differences between dataset sizes and hence inconsistent numbers of updates + # per 'epoch'. + optimizer = torch.optim.Adam( + self.net.parameters(), lr=self.config.clients.CNN.lr, betas=(0.5, 0.999) + ) + metric_fn = instantiate(self.config.dataset.task.metric.fn) + criterion = instantiate(self.config.dataset.task.criterion) + for _i, data in zip(range(num_iterations), trainloader): + loss, metric_val, n_samples = self.train_one_loop( + data, optimizer, metric_fn, criterion + ) + total_loss += loss + total_result += metric_val + total_n_samples += n_samples + + return ( + total_loss / total_n_samples, + total_result / total_n_samples, + total_n_samples, + ) + + def test(self, net: CNN, testloader: DataLoader) -> Tuple[float, float, int]: + """Evaluates the network on test data. + + Parameters + ---------- + net: The CNN model to be tested. + testloader: The data loader containing the test data. + + Return: A tuple containing the average loss, + average metric result, + and the total number of samples tested. + """ + total_loss, total_result, n_samples = 0.0, 0.0, 0 + net.eval() + metric_fn = instantiate(self.config.dataset.task.metric.fn) + criterion = instantiate(self.config.dataset.task.criterion) + with torch.no_grad(): + for data in testloader: + tree_outputs, labels = data[0].to(self.device), data[1].to(self.device) + outputs = net(tree_outputs) + total_loss += criterion(outputs, labels).item() + n_samples += labels.size(0) + metric_val = metric_fn(outputs.cpu(), labels.type(torch.int).cpu()) + total_result += metric_val * labels.size(0) + + return total_loss / n_samples, total_result / n_samples, n_samples + + def get_parameters(self, ins): + """Get CNN net weights and the tree. + + Parameters + ---------- + - self (object): The instance of the class that the function belongs to. + - ins (GetParametersIns): An input parameter object. + + Returns + ------- + Tuple[GetParametersRes, + Union[Tuple[XGBClassifier, int],Tuple[XGBRegressor, int]]]: + A tuple containing the parameters of the net and the tree. + - GetParametersRes: + - status : An object with the status code. + - parameters : An ndarray containing the model's weights. + - Union[Tuple[XGBClassifier, int], Tuple[XGBRegressor, int]]: + A tuple containing either an XGBClassifier or XGBRegressor + object along with client's id. + """ + for dataset in self.trainloader_original: + data, label = dataset[0], dataset[1] + + tree = fit_xgboost( + self.config, self.config.dataset.task.task_type, data, label, 100 + ) + return GetParametersRes( + status=Status(Code.OK, ""), + parameters=ndarrays_to_parameters(self.net.get_weights()), + ), (tree, int(self.cid)) + + def fit(self, ins: FitIns) -> FitRes: + """Trains a model using the given fit parameters. + + Parameters + ---------- + ins: FitIns - The fit parameters that contain the configuration + and parameters needed for training. + + Returns + ------- + FitRes - An object that contains the status, trained parameters, + number of examples processed, and metrics. + """ + num_iterations = ins.config["num_iterations"] + batch_size = ins.config["batch_size"] + + # set parmeters + self.net.set_weights(parameters_to_ndarrays(ins.parameters[0])) # type: ignore # noqa: E501 # pylint: disable=line-too-long + aggregated_trees = ins.parameters[1] # type: ignore # noqa: E501 # pylint: disable=line-too-long + + if isinstance(aggregated_trees, list): + print("Client " + self.cid + ": recieved", len(aggregated_trees), "trees") + else: + print("Client " + self.cid + ": only had its own tree") + trainloader: Any = single_tree_preds_from_each_client( + self.trainloader_original, + batch_size, + aggregated_trees, + self.config.n_estimators_client, + self.config.clients.client_num, + ) + self.valloader = single_tree_preds_from_each_client( + self.valloader_original, + batch_size, + aggregated_trees, + self.config.n_estimators_client, + self.config.clients.client_num, + ) + + # runs for a single epoch, however many updates it may be + num_iterations = int(num_iterations) or len(trainloader) + # Train the model + print( + "Client", self.cid, ": training for", num_iterations, "iterations/updates" + ) + self.net.to(self.device) + train_loss, train_result, num_examples = self.train( + self.net, + trainloader, + num_iterations=num_iterations, + ) + print( + f"Client {self.cid}: training round complete, {num_examples}", + "examples processed", + ) + + # Return training information: model, number of examples processed and metrics + return FitRes( + status=Status(Code.OK, ""), + parameters=self.get_parameters(ins.config), + num_examples=num_examples, + metrics={ + "loss": train_loss, + self.config.dataset.task.metric.name: train_result, + }, + ) + + def evaluate(self, ins: EvaluateIns) -> EvaluateRes: + """Evaluate CNN model using the given evaluation parameters. + + Parameters + ---------- + ins: An instance of EvaluateIns class that contains the parameters + for evaluation. + Return: + An EvaluateRes object that contains the evaluation results. + """ + # set the weights of the CNN net + self.net.set_weights(parameters_to_ndarrays(ins.parameters)) + + # Evaluate the model + self.net.to(self.device) + loss, result, num_examples = self.test( + self.net, + self.valloader, + ) + + # Return evaluation information + print( + f"Client {self.cid}: evaluation on {num_examples} examples:", + f"loss={loss:.4f}", + self.config.dataset.task.metric.name, + f"={result:.4f}", + ) + return EvaluateRes( + status=Status(Code.OK, ""), + loss=loss, + num_examples=num_examples, + metrics={self.config.dataset.task.metric.name: result}, + ) diff --git a/baselines/hfedxgboost/hfedxgboost/conf/Centralized_Baseline.yaml b/baselines/hfedxgboost/hfedxgboost/conf/Centralized_Baseline.yaml new file mode 100644 index 000000000000..aac3f68bbe51 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/Centralized_Baseline.yaml @@ -0,0 +1,22 @@ +centralized: True +defaults: + - dataset: abalone + - xgboost_params_centralized: abalone_xgboost_centralized + +n_estimators_client: ${xgboost_params_centralized.n_estimators} +task_type: ${dataset.task.task_type} + +XGBoost: + _target_: ${dataset.task.xgb._target_} + objective: ${dataset.task.xgb.objective} + learning_rate: ${xgboost_params_centralized.learning_rate} + max_depth: ${xgboost_params_centralized.max_depth} + n_estimators: ${xgboost_params_centralized.n_estimators} + subsample: ${xgboost_params_centralized.subsample} + colsample_bylevel: ${xgboost_params_centralized.colsample_bylevel} + colsample_bynode: ${xgboost_params_centralized.colsample_bynode} + colsample_bytree: ${xgboost_params_centralized.colsample_bytree} + alpha: ${xgboost_params_centralized.alpha} + gamma: ${xgboost_params_centralized.gamma} + num_parallel_tree: ${xgboost_params_centralized.num_parallel_tree} + min_child_weight: ${xgboost_params_centralized.min_child_weight} diff --git a/baselines/hfedxgboost/hfedxgboost/conf/base.yaml b/baselines/hfedxgboost/hfedxgboost/conf/base.yaml new file mode 100644 index 000000000000..310b123a9054 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/base.yaml @@ -0,0 +1,55 @@ +--- +defaults: + - dataset: cpusmall + - clients: cpusmall_5_clients + - wandb: default + +centralized: False +use_wandb: False +show_each_client_performance_on_its_local_data: False +val_ratio: 0.0 +batch_size: "whole" +n_estimators_client: ${clients.n_estimators_client} +task_type: ${dataset.task.task_type} +client_num: ${clients.client_num} + +XGBoost: + _target_: ${dataset.task.xgb._target_} + objective: ${dataset.task.xgb.objective} + learning_rate: .1 + max_depth: ${clients.xgb.max_depth} + n_estimators: ${clients.n_estimators_client} + subsample: 0.8 + colsample_bylevel: 1 + colsample_bynode: 1 + colsample_bytree: 1 + alpha: 5 + gamma: 5 + num_parallel_tree: 1 + min_child_weight: 1 + +server: + max_workers: None + device: "cpu" + +client_resources: + num_cpus: 1 + num_gpus: 0.0 + +strategy: + _target_: flwr.server.strategy.FedXgbNnAvg + _recursive_: true #everything to be instantiated + fraction_fit: 1.0 + fraction_evaluate: 0.0 # no clients will be sampled for federated evaluation (we will still perform global evaluation) + min_fit_clients: 1 + min_evaluate_clients: 1 + min_available_clients: ${client_num} + accept_failures: False + +run_experiment: + num_rounds: ${clients.num_rounds} + batch_size: 32 + fraction_fit: 1.0 + min_fit_clients: 1 + fit_config: + num_iterations: ${clients.num_iterations} diff --git a/baselines/hfedxgboost/hfedxgboost/conf/centralized_basline_all_datasets_paper_config.yaml b/baselines/hfedxgboost/hfedxgboost/conf/centralized_basline_all_datasets_paper_config.yaml new file mode 100644 index 000000000000..51d168021ac9 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/centralized_basline_all_datasets_paper_config.yaml @@ -0,0 +1,46 @@ +centralized: True +learning_rate: 0.1 +max_depth: 8 +n_estimators: 500 +subsample: 0.8 +colsample_bylevel: 1 +colsample_bynode: 1 +colsample_bytree: 1 +alpha: 5 +gamma: 5 +num_parallel_tree: 1 +min_child_weight: 1 + +dataset: + dataset_name: "all" + train_ratio: .75 + +XGBoost: + classifier: + _target_: xgboost.XGBClassifier + objective: "binary:logistic" + learning_rate: ${learning_rate} + max_depth: ${max_depth} + n_estimators: ${n_estimators} + subsample: ${subsample} + colsample_bylevel: ${colsample_bylevel} + colsample_bynode: ${colsample_bynode} + colsample_bytree: ${colsample_bytree} + alpha: ${alpha} + gamma: ${gamma} + num_parallel_tree: ${num_parallel_tree} + min_child_weight: ${min_child_weight} + regressor: + _target_: xgboost.XGBRegressor + objective: "reg:squarederror" + learning_rate: ${learning_rate} + max_depth: ${max_depth} + n_estimators: ${n_estimators} + subsample: ${subsample} + colsample_bylevel: ${colsample_bylevel} + colsample_bynode: ${colsample_bynode} + colsample_bytree: ${colsample_bytree} + alpha: ${alpha} + gamma: ${gamma} + num_parallel_tree: ${num_parallel_tree} + min_child_weight: ${min_child_weight} diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_10_clients.yaml new file mode 100644 index 000000000000..48fef3d2dd57 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 50 +num_rounds: 200 +client_num: 10 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_2_clients.yaml new file mode 100644 index 000000000000..d960e5ee5f40 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 250 +num_rounds: 200 +client_num: 2 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: 0.0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_5_clients.yaml new file mode 100644 index 000000000000..7e807e873b17 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/YearPredictionMSD_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 200 +client_num: 5 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_10_clients.yaml new file mode 100644 index 000000000000..4839ccb6dc91 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 10 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_2_clients.yaml new file mode 100644 index 000000000000..f38cf782a239 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 250 +num_rounds: 15 +client_num: 2 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_5_clients.yaml new file mode 100644 index 000000000000..c331db5e258a --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/a9a_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 5 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0005 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_10_clients.yaml new file mode 100644 index 000000000000..055db27bef85 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 25 +num_rounds: 200 +client_num: 10 +num_iterations: 100 + +xgb: + max_depth: 6 +CNN: + lr: .0006301009302952918 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_2_clients.yaml new file mode 100644 index 000000000000..e4fecac5fb4e --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 50 +num_rounds: 100 +client_num: 2 +num_iterations: 100 + +xgb: + max_depth: 6 +CNN: + lr: 0.0028231080803766 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_5_clients.yaml new file mode 100644 index 000000000000..0610209d0b70 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/abalone_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 50 +num_rounds: 200 +client_num: 5 +num_iterations: 500 + +xgb: + max_depth: 6 +CNN: + lr: .0004549072000953885 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_10_clients.yaml new file mode 100644 index 000000000000..4839ccb6dc91 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 10 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_2_clients.yaml new file mode 100644 index 000000000000..9270ae839675 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 250 +num_rounds: 30 +client_num: 2 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_5_clients.yaml new file mode 100644 index 000000000000..9237b5c4362a --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cod_rna_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 5 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_10_clients.yaml new file mode 100644 index 000000000000..e6882134ec84 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 50 +num_rounds: 1000 +num_iterations: 500 +client_num: 10 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_2_clients.yaml new file mode 100644 index 000000000000..bd8552875412 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 1000 +client_num: 2 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_5_clients.yaml new file mode 100644 index 000000000000..31a740714b2a --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/cpusmall_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 25 +num_rounds: 500 +client_num: 5 +num_iterations: 100 + +xgb: + max_depth: 6 +CNN: + lr: .000457414512764587 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_10_clients.yaml new file mode 100644 index 000000000000..cf96d4e5c394 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 10 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0005 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_2_clients.yaml new file mode 100644 index 000000000000..69bbf5b26701 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 250 +num_rounds: 50 +client_num: 2 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_5_clients.yaml new file mode 100644 index 000000000000..945dbe885345 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/ijcnn1_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 30 +client_num: 5 +num_iterations: 500 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_10_clients.yaml new file mode 100644 index 000000000000..08076f993a4c --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 50 +num_rounds: 100 +client_num: 10 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_2_clients.yaml new file mode 100644 index 000000000000..96802df2c193 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 250 +num_rounds: 100 +client_num: 2 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_5_clients.yaml new file mode 100644 index 000000000000..cec270b5a52d --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/paper_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 100 +client_num: 5 +num_iterations: 100 + +xgb: + max_depth: 8 +CNN: + lr: .0001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_10_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_10_clients.yaml new file mode 100644 index 000000000000..ede979063464 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_10_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 50 +client_num: 10 +num_iterations: 500 + +xgb: + max_depth: 4 +CNN: + lr: .00001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_2_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_2_clients.yaml new file mode 100644 index 000000000000..a9a2b8bb38a9 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_2_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 50 +client_num: 2 +num_iterations: 500 + +xgb: + max_depth: 4 +CNN: + lr: .00001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_5_clients.yaml b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_5_clients.yaml new file mode 100644 index 000000000000..f10de46665c1 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/clients/space_ga_5_clients.yaml @@ -0,0 +1,9 @@ +n_estimators_client: 100 +num_rounds: 200 +client_num: 5 +num_iterations: 500 + +xgb: + max_depth: 4 +CNN: + lr: .000001 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/YearPredictionMSD.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/YearPredictionMSD.yaml new file mode 100644 index 000000000000..b7d7ed14bfd6 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/YearPredictionMSD.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Regression +dataset_name: "YearPredictionMSD" +train_ratio: .75 +early_stop_patience_rounds: 50 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/a9a.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/a9a.yaml new file mode 100644 index 000000000000..ded8e4fe40c7 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/a9a.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Binary_Classification +dataset_name: "a9a" +train_ratio: .75 +early_stop_patience_rounds: 10 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/abalone.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/abalone.yaml new file mode 100644 index 000000000000..6aad6c65a068 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/abalone.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Regression +dataset_name: "abalone" +train_ratio: .75 +early_stop_patience_rounds: 30 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/cod_rna.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/cod_rna.yaml new file mode 100644 index 000000000000..ea6e68554f80 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/cod_rna.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Binary_Classification +dataset_name: "cod-rna" +train_ratio: .75 +early_stop_patience_rounds: 10 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/cpusmall.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/cpusmall.yaml new file mode 100644 index 000000000000..6aeec735b4b0 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/cpusmall.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Regression +dataset_name: "cpusmall" +train_ratio: .75 +early_stop_patience_rounds: 100 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/ijcnn1.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/ijcnn1.yaml new file mode 100644 index 000000000000..0678f04d0d69 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/ijcnn1.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Binary_Classification +dataset_name: "ijcnn1" +train_ratio: .75 +early_stop_patience_rounds: 10 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/space_ga.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/space_ga.yaml new file mode 100644 index 000000000000..fa89f8e852bd --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/space_ga.yaml @@ -0,0 +1,5 @@ +defaults: + - task: Regression +dataset_name: "space_ga" +train_ratio: .75 +early_stop_patience_rounds: 50 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Binary_Classification.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Binary_Classification.yaml new file mode 100644 index 000000000000..f4e282e181bf --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Binary_Classification.yaml @@ -0,0 +1,14 @@ +task_type: "BINARY" + +metric: + name: "Accuracy" + fn: + _target_: torchmetrics.Accuracy + task: "binary" + +criterion: + _target_: torch.nn.BCELoss + +xgb: + _target_: xgboost.XGBClassifier + objective: "binary:logistic" diff --git a/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Regression.yaml b/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Regression.yaml new file mode 100644 index 000000000000..37fdca8894c3 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/dataset/task/Regression.yaml @@ -0,0 +1,14 @@ +task_type: "REG" + +metric: + name: "mse" + fn: + _target_: torchmetrics.MeanSquaredError + +criterion: + _target_: torch.nn.MSELoss + + +xgb: + _target_: xgboost.XGBRegressor + objective: "reg:squarederror" \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/wandb/default.yaml b/baselines/hfedxgboost/hfedxgboost/conf/wandb/default.yaml new file mode 100644 index 000000000000..36968201fdc6 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/wandb/default.yaml @@ -0,0 +1,6 @@ +setup: + project: p1 + mode: online +watch: + log: all + log_freq: 100 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/YearPredictionMSD_xgboost_centralized.yaml b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/YearPredictionMSD_xgboost_centralized.yaml new file mode 100644 index 000000000000..1912c7d5a015 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/YearPredictionMSD_xgboost_centralized.yaml @@ -0,0 +1,11 @@ +n_estimators: 300 +max_depth: 4 +subsample: 0.8 +learning_rate: .1 +colsample_bylevel: 1 +colsample_bynode: 1 +colsample_bytree: 1 +alpha: 5 +gamma: 5 +num_parallel_tree: 1 +min_child_weight: 1 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/abalone_xgboost_centralized.yaml b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/abalone_xgboost_centralized.yaml new file mode 100644 index 000000000000..72578770034d --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/abalone_xgboost_centralized.yaml @@ -0,0 +1,11 @@ +n_estimators: 200 +max_depth: 3 +subsample: .4 +learning_rate: .05 +colsample_bylevel: 1 +colsample_bynode: 1 +colsample_bytree: 1 +alpha: 5 +gamma: 10 +num_parallel_tree: 1 +min_child_weight: 5 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/cpusmall_xgboost_centralized.yaml b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/cpusmall_xgboost_centralized.yaml new file mode 100644 index 000000000000..33983403091d --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/cpusmall_xgboost_centralized.yaml @@ -0,0 +1,11 @@ +n_estimators: 10000 +max_depth: 3 +subsample: .4 +learning_rate: .05 +colsample_bylevel: .5 +colsample_bynode: 1 +colsample_bytree: 1 +alpha: 5 +gamma: 10 +num_parallel_tree: 1 +min_child_weight: 5 diff --git a/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/paper_xgboost_centralized.yaml b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/paper_xgboost_centralized.yaml new file mode 100644 index 000000000000..f439badb3ade --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/conf/xgboost_params_centralized/paper_xgboost_centralized.yaml @@ -0,0 +1,11 @@ +n_estimators: 500 +max_depth: 8 +subsample: 0.8 +learning_rate: .1 +colsample_bylevel: 1 +colsample_bynode: 1 +colsample_bytree: 1 +alpha: 5 +gamma: 5 +num_parallel_tree: 1 +min_child_weight: 1 \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/dataset.py b/baselines/hfedxgboost/hfedxgboost/dataset.py new file mode 100644 index 000000000000..a03ce2cd59fa --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/dataset.py @@ -0,0 +1,142 @@ +"""Handle basic dataset creation. + +In case of PyTorch it should return dataloaders for your dataset (for both the clients +and the server). If you are using a custom dataset class, this module is the place to +define it. If your dataset requires to be downloaded (and this is not done +automatically -- e.g. as it is the case for many dataset in TorchVision) and +partitioned, please include all those functions and logic in the +`dataset_preparation.py` module. You can use all those functions from functions/methods +defined here of course. +""" +from typing import List, Optional, Tuple, Union + +import torch +from flwr.common import NDArray +from torch.utils.data import DataLoader, Dataset, random_split + +from hfedxgboost.dataset_preparation import ( + datafiles_fusion, + download_data, + modify_labels, + train_test_split, +) + + +def load_single_dataset( + task_type: str, dataset_name: str, train_ratio: Optional[float] = 0.75 +) -> Tuple[NDArray, NDArray, NDArray, NDArray]: + """Load a single dataset. + + Parameters + ---------- + task_type (str): The type of task, either "BINARY" or "REG". + dataset_name (str): The name of the dataset to load. + train_ratio (float, optional): The ratio of training data to the total dataset. + Default is 0.75. + + Returns + ------- + x_train (numpy array): The training data features. + y_train (numpy array): The training data labels. + X_test (numpy array): The testing data features. + y_test (numpy array): The testing data labels. + """ + datafiles_paths = download_data(dataset_name) + X, Y = datafiles_fusion(datafiles_paths) + x_train, y_train, x_test, y_test = train_test_split(X, Y, train_ratio=train_ratio) + if task_type.upper() == "BINARY": + y_train, y_test = modify_labels(y_train, y_test) + + print( + "First class ratio in train data", + y_train[y_train == 0.0].size / x_train.shape[0], + ) + print( + "Second class ratio in train data", + y_train[y_train != 0.0].size / x_train.shape[0], + ) + print( + "First class ratio in test data", + y_test[y_test == 0.0].size / x_test.shape[0], + ) + print( + "Second class ratio in test data", + y_test[y_test != 0.0].size / x_test.shape[0], + ) + + print("Feature dimension of the dataset:", x_train.shape[1]) + print("Size of the trainset:", x_train.shape[0]) + print("Size of the testset:", x_test.shape[0]) + + return x_train, y_train, x_test, y_test + + +def get_dataloader( + dataset: Dataset, partition: str, batch_size: Union[int, str] +) -> DataLoader: + """Return a DataLoader object. + + Parameters + ---------- + dataset (Dataset): The dataset object that contains the data. + partition (str): The partition string that specifies the subset of data to use. + batch_size (Union[int, str]): The batch size to use for loading data. + It can be either an integer value or the string "whole". + If "whole" is provided, the batch size will be set to the length of the dataset. + + Returns + ------- + DataLoader: A DataLoader object that loads data from the dataset in batches. + """ + if batch_size == "whole": + batch_size = len(dataset) + return DataLoader( + dataset, batch_size=batch_size, pin_memory=True, shuffle=(partition == "train") + ) + + +def divide_dataset_between_clients( + trainset: Dataset, + testset: Dataset, + pool_size: int, + batch_size: Union[int, str], + val_ratio: float = 0.0, +) -> Tuple[DataLoader, Union[List[DataLoader], List[None]], DataLoader]: + """Divide the data between clients with IID distribution. + + Parameters + ---------- + trainset (Dataset): The full training dataset. + testset (Dataset): The full test dataset. + pool_size (int): The number of partitions to create. + batch_size (Union[int, str]): The size of the batches. + val_ratio (float, optional): The ratio of validation data. Defaults to 0.0. + + Returns + ------- + Tuple[DataLoader, DataLoader, DataLoader]: A tuple containing + the training loaders, validation loaders (or None), and test loader. + """ + # Split training set into `num_clients` partitions to simulate + # different local datasets + trainset_length = len(trainset) + lengths = [trainset_length // pool_size] * pool_size + if sum(lengths) != trainset_length: + lengths[-1] = trainset_length - sum(lengths[0:-1]) + datasets = random_split(trainset, lengths, torch.Generator().manual_seed(0)) + + # Split each partition into train/val and create DataLoader + trainloaders: List[DataLoader] = [] + valloaders: Union[List[DataLoader], List[None]] = [] + for dataset in datasets: + len_val = int(len(dataset) * val_ratio) + len_train = len(dataset) - len_val + ds_train, ds_val = random_split( + dataset, [len_train, len_val], torch.Generator().manual_seed(0) + ) + trainloaders.append(get_dataloader(ds_train, "train", batch_size)) + if len_val != 0: + valloaders.append(get_dataloader(ds_val, "val", batch_size)) + else: + valloaders.append(None) + return trainloaders, valloaders, get_dataloader(testset, "test", batch_size) diff --git a/baselines/hfedxgboost/hfedxgboost/dataset_preparation.py b/baselines/hfedxgboost/hfedxgboost/dataset_preparation.py new file mode 100644 index 000000000000..3fd3cbfb68fd --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/dataset_preparation.py @@ -0,0 +1,262 @@ +"""Handle the dataset partitioning and (optionally) complex downloads. + +Please add here all the necessary logic to either download, uncompress, pre/post-process +your dataset (or all of the above). If the desired way of running your baseline is to +first download the dataset and partition it and then run the experiments, please +uncomment the lines below and tell us in the README.md (see the "Running the Experiment" +block) that this file should be executed first. +""" +import bz2 +import os +import shutil +import urllib.request +from typing import Optional + +import numpy as np +from sklearn.datasets import load_svmlight_file + + +def download_data(dataset_name: Optional[str] = "cod-rna"): + """Download (if necessary) the dataset and returns the dataset path. + + Parameters + ---------- + dataset_name : String + A string stating the name of the dataset that need to be dowenloaded. + + Returns + ------- + List[Dataset Pathes] + The pathes for the data that will be used in train and test, + with train of full dataset in index 0 + """ + all_datasets_path = "./dataset" + if dataset_name: + dataset_path = os.path.join(all_datasets_path, dataset_name) + match dataset_name: + case "a9a": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/a9a", + f"{os.path.join(dataset_path, 'a9a')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/a9a.t", + f"{os.path.join(dataset_path, 'a9a.t')}", + ) + # training then test ✅ + return_list = [ + os.path.join(dataset_path, "a9a"), + os.path.join(dataset_path, "a9a.t"), + ] + case "cod-rna": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/cod-rna.t", + f"{os.path.join(dataset_path, 'cod-rna.t')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/cod-rna.r", + f"{os.path.join(dataset_path, 'cod-rna.r')}", + ) + # training then test ✅ + return_list = [ + os.path.join(dataset_path, "cod-rna.t"), + os.path.join(dataset_path, "cod-rna.r"), + ] + + case "ijcnn1": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/ijcnn1.bz2", + f"{os.path.join(dataset_path, 'ijcnn1.tr.bz2')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/ijcnn1.t.bz2", + f"{os.path.join(dataset_path, 'ijcnn1.t.bz2')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/ijcnn1.tr.bz2", + f"{os.path.join(dataset_path, 'ijcnn1.tr.bz2')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/binary/ijcnn1.val.bz2", + f"{os.path.join(dataset_path, 'ijcnn1.val.bz2')}", + ) + + for filepath in os.listdir(dataset_path): + abs_filepath = os.path.join(dataset_path, filepath) + with bz2.BZ2File(abs_filepath) as freader, open( + abs_filepath[:-4], "wb" + ) as fwriter: + shutil.copyfileobj(freader, fwriter) + # training then test ✅ + return_list = [ + os.path.join(dataset_path, "ijcnn1.t"), + os.path.join(dataset_path, "ijcnn1.tr"), + ] + + case "space_ga": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/regression/space_ga_scale", + f"{os.path.join(dataset_path, 'space_ga_scale')}", + ) + return_list = [os.path.join(dataset_path, "space_ga_scale")] + case "abalone": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/regression/abalone_scale", + f"{os.path.join(dataset_path, 'abalone_scale')}", + ) + return_list = [os.path.join(dataset_path, "abalone_scale")] + case "cpusmall": + if not os.path.exists(dataset_path): + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/regression/cpusmall_scale", + f"{os.path.join(dataset_path, 'cpusmall_scale')}", + ) + return_list = [os.path.join(dataset_path, "cpusmall_scale")] + case "YearPredictionMSD": + if not os.path.exists(dataset_path): + print( + "long download coming -~615MB-, it'll be better if you downloaded", + "those 2 files manually with a faster download manager program or", + "something and just place them in the right folder then get", + "the for loop out of the if condition to alter their format", + ) + os.makedirs(dataset_path) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/regression/YearPredictionMSD.bz2", + f"{os.path.join(dataset_path, 'YearPredictionMSD.bz2')}", + ) + urllib.request.urlretrieve( + "https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets" + "/regression/YearPredictionMSD.t.bz2", + f"{os.path.join(dataset_path, 'YearPredictionMSD.t.bz2')}", + ) + for filepath in os.listdir(dataset_path): + print("it will take sometime") + abs_filepath = os.path.join(dataset_path, filepath) + with bz2.BZ2File(abs_filepath) as freader, open( + abs_filepath[:-4], "wb" + ) as fwriter: + shutil.copyfileobj(freader, fwriter) + return_list = [ + os.path.join(dataset_path, "YearPredictionMSD"), + os.path.join(dataset_path, "YearPredictionMSD.t"), + ] + case _: + raise Exception("write your own dataset downloader") + return return_list + + +def datafiles_fusion(data_paths): + """Merge (if necessary) the data files and returns the features and labels. + + Parmetres: + data_paths: List[Dataset Pathes] + - The pathes for the data that will be used in train and test, + with train of full dataset in index 0 + Returns: + X: Numpy array + - The full features of the dataset. + y: Numpy array + - The full labels of the dataset. + """ + data = load_svmlight_file(data_paths[0], zero_based=False) + X = data[0].toarray() + Y = data[1] + for i in range(1, len(data_paths)): + data = load_svmlight_file( + data_paths[i], zero_based=False, n_features=X.shape[1] + ) + X = np.concatenate((X, data[0].toarray()), axis=0) + Y = np.concatenate((Y, data[1]), axis=0) + return X, Y + + +def train_test_split(X, y, train_ratio=0.75): + """Split the dataset into training and testing. + + Parameters + ---------- + X: Numpy array + The full features of the dataset. + y: Numpy array + The full labels of the dataset. + train_ratio: float + the ratio that training should take from the full dataset + + Returns + ------- + X_train: Numpy array + The training dataset features. + y_train: Numpy array + The labels of the training dataset. + X_test: Numpy array + The testing dataset features. + y_test: Numpy array + The labels of the testing dataset. + """ + np.random.seed(2023) + y = np.expand_dims(y, axis=1) + full = np.concatenate((X, y), axis=1) + np.random.shuffle(full) + y = full[:, -1] # for last column + X = full[:, :-1] # for all but last column + num_training_samples = int(X.shape[0] * train_ratio) + + x_train = X[0:num_training_samples] + y_train = y[0:num_training_samples] + + x_test = X[num_training_samples:] + y_test = y[num_training_samples:] + + x_train.flags.writeable = True + y_train.flags.writeable = True + x_test.flags.writeable = True + y_test.flags.writeable = True + + return x_train, y_train, x_test, y_test + + +def modify_labels(y_train, y_test): + """Switch the -1 in the classification dataset with 0. + + Parameters + ---------- + y_train: Numpy array + The labels of the training dataset. + y_test: Numpy array + The labels of the testing dataset. + + Returns + ------- + y_train: Numpy array + The labels of the training dataset. + y_test: Numpy array + The labels of the testing dataset. + """ + y_train[y_train == -1] = 0 + y_test[y_test == -1] = 0 + return y_train, y_test diff --git a/baselines/hfedxgboost/hfedxgboost/main.py b/baselines/hfedxgboost/hfedxgboost/main.py new file mode 100644 index 000000000000..061e635f024c --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/main.py @@ -0,0 +1,143 @@ +"""Create and connect the building blocks for your experiments; start the simulation. + +It includes processioning the dataset, instantiate strategy, specify how the global +model is going to be evaluated, etc. At the end, this script saves the results. +""" +import functools +from typing import Dict, Union + +import flwr as fl +import hydra +import torch +import wandb +from flwr.common import Scalar +from flwr.server.app import ServerConfig +from flwr.server.client_manager import SimpleClientManager +from hydra.utils import instantiate +from omegaconf import DictConfig, OmegaConf +from torch.utils.data import TensorDataset + +from hfedxgboost.client import FlClient +from hfedxgboost.dataset import divide_dataset_between_clients, load_single_dataset +from hfedxgboost.server import FlServer, serverside_eval +from hfedxgboost.utils import ( + CentralizedResultsWriter, + EarlyStop, + ResultsWriter, + create_res_csv, + local_clients_performance, + run_centralized, +) + + +@hydra.main(config_path="conf", config_name="base", version_base=None) +def main(cfg: DictConfig) -> None: + """Run the baseline. + + Parameters + ---------- + cfg : DictConfig + An omegaconf object that stores the hydra config. + """ + # 1. Print parsed config + print(OmegaConf.to_yaml(cfg)) + writer: Union[ResultsWriter, CentralizedResultsWriter] + if cfg.centralized: + if cfg.dataset.dataset_name == "all": + run_centralized(cfg, dataset_name=cfg.dataset.dataset_name) + else: + writer = CentralizedResultsWriter(cfg) + create_res_csv("results_centralized.csv", writer.fields) + writer.write_res( + "results_centralized.csv", + run_centralized(cfg, dataset_name=cfg.dataset.dataset_name)[0], + run_centralized(cfg, dataset_name=cfg.dataset.dataset_name)[1], + ) + else: + if cfg.use_wandb: + wandb.init(**cfg.wandb.setup, group=f"{cfg.dataset.dataset_name}") + + print("Dataset Name", cfg.dataset.dataset_name) + early_stopper = EarlyStop(cfg) + x_train, y_train, x_test, y_test = load_single_dataset( + cfg.dataset.task.task_type, + cfg.dataset.dataset_name, + train_ratio=cfg.dataset.train_ratio, + ) + + trainloaders, valloaders, testloader = divide_dataset_between_clients( + TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)), + TensorDataset(torch.from_numpy(x_test), torch.from_numpy(y_test)), + batch_size=cfg.batch_size, + pool_size=cfg.clients.client_num, + val_ratio=cfg.val_ratio, + ) + print( + f"Data partitioned across {cfg.clients.client_num} clients" + f" and {cfg.val_ratio} of local dataset reserved for validation." + ) + if cfg.show_each_client_performance_on_its_local_data: + local_clients_performance( + cfg, trainloaders, x_test, y_test, cfg.dataset.task.task_type + ) + + # Configure the strategy + def fit_config(server_round: int) -> Dict[str, Scalar]: + print(f"Configuring round {server_round}") + return { + "num_iterations": cfg.run_experiment.fit_config.num_iterations, + "batch_size": cfg.run_experiment.batch_size, + } + + # FedXgbNnAvg + strategy = instantiate( + cfg.strategy, + on_fit_config_fn=fit_config, + on_evaluate_config_fn=( + lambda r: {"batch_size": cfg.run_experiment.batch_size} + ), + evaluate_fn=functools.partial( + serverside_eval, + cfg=cfg, + testloader=testloader, + ), + ) + + print( + f"FL experiment configured for {cfg.run_experiment.num_rounds} rounds with", + f"{cfg.clients.client_num} client in the pool.", + ) + + def client_fn(cid: str) -> fl.client.Client: + """Create a federated learning client.""" + return FlClient(cfg, trainloaders[int(cid)], valloaders[int(cid)], cid) + + # Start the simulation + history = fl.simulation.start_simulation( + client_fn=client_fn, + server=FlServer( + cfg=cfg, + client_manager=SimpleClientManager(), + early_stopper=early_stopper, + strategy=strategy, + ), + num_clients=cfg.clients.client_num, + client_resources=cfg.client_resources, + config=ServerConfig(num_rounds=cfg.run_experiment.num_rounds), + strategy=strategy, + ) + + print(history) + writer = ResultsWriter(cfg) + print( + "Best Result", + writer.extract_best_res(history)[0], + "best_res_round", + writer.extract_best_res(history)[1], + ) + create_res_csv("results.csv", writer.fields) + writer.write_res("results.csv") + + +if __name__ == "__main__": + main() diff --git a/baselines/hfedxgboost/hfedxgboost/models.py b/baselines/hfedxgboost/hfedxgboost/models.py new file mode 100644 index 000000000000..fbfc2d966f69 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/models.py @@ -0,0 +1,130 @@ +"""Define our models, and training and eval functions. + +If your model is 100% off-the-shelf (e.g. directly from torchvision without requiring +modifications) you might be better off instantiating your model directly from the Hydra +config. In this way, swapping your model for another one can be done without changing +the python code at all +""" +from collections import OrderedDict +from typing import Union + +import flwr as fl +import numpy as np +import torch +import torch.nn as nn +from flwr.common import NDArray +from hydra.utils import instantiate +from omegaconf import DictConfig +from xgboost import XGBClassifier, XGBRegressor + + +def fit_xgboost( + config: DictConfig, + task_type: str, + x_train: NDArray, + y_train: NDArray, + n_estimators: int, +) -> Union[XGBClassifier, XGBRegressor]: + """Fit XGBoost model to training data. + + Parameters + ---------- + config (DictConfig): Hydra configuration. + task_type (str): Type of task, "REG" for regression and "BINARY" + for binary classification. + x_train (NDArray): Input features for training. + y_train (NDArray): Labels for training. + n_estimators (int): Number of trees to build. + + Returns + ------- + Union[XGBClassifier, XGBRegressor]: Fitted XGBoost model. + """ + if config.dataset.dataset_name == "all": + if task_type.upper() == "REG": + tree = instantiate(config.XGBoost.regressor, n_estimators=n_estimators) + elif task_type.upper() == "BINARY": + tree = instantiate(config.XGBoost.classifier, n_estimators=n_estimators) + else: + tree = instantiate(config.XGBoost) + tree.fit(x_train, y_train) + return tree + + +class CNN(nn.Module): + """CNN model.""" + + def __init__(self, cfg: DictConfig, n_channel: int = 64) -> None: + super().__init__() + n_out = 1 + self.task_type = cfg.dataset.task.task_type + n_estimators_client = cfg.n_estimators_client + client_num = cfg.client_num + + self.conv1d = nn.Conv1d( + in_channels=1, + out_channels=n_channel, + kernel_size=n_estimators_client, + stride=n_estimators_client, + padding=0, + ) + + self.layer_direct = nn.Linear(n_channel * client_num, n_out) + + self.relu = nn.ReLU() + + if self.task_type == "BINARY": + self.final_layer = nn.Sigmoid() + elif self.task_type == "REG": + self.final_layer = nn.Identity() + + # Add weight initialization + for layer in self.modules(): + if isinstance(layer, nn.Linear): + nn.init.kaiming_uniform_( + layer.weight, mode="fan_in", nonlinearity="relu" + ) + + def forward(self, input_features: torch.Tensor) -> torch.Tensor: + """Perform a forward pass. + + Parameters + ---------- + input_features (torch.Tensor): Input features to the network. + + Returns + ------- + output (torch.Tensor): Output of the network after the forward pass. + """ + output = self.conv1d(input_features) + output = output.flatten(start_dim=1) + output = self.relu(output) + output = self.layer_direct(output) + output = self.final_layer(output) + return output + + def get_weights(self) -> fl.common.NDArrays: + """Get model weights. + + Parameters + ---------- + a list of NumPy arrays. + """ + return [ + np.array(val.cpu().numpy(), copy=True) + for _, val in self.state_dict().items() + ] + + def set_weights(self, weights: fl.common.NDArrays) -> None: + """Set the CNN model weights. + + Parameters + ---------- + weights:a list of NumPy arrays + """ + layer_dict = {} + for key, value in zip(self.state_dict().keys(), weights): + if value.ndim != 0: + layer_dict[key] = torch.Tensor(np.array(value, copy=True)) + state_dict = OrderedDict(layer_dict) + self.load_state_dict(state_dict, strict=True) diff --git a/baselines/hfedxgboost/hfedxgboost/server.py b/baselines/hfedxgboost/hfedxgboost/server.py new file mode 100644 index 000000000000..e8ac6a24add8 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/server.py @@ -0,0 +1,406 @@ +"""A custom server class extends Flower's default server class to build a federated. + +learning setup that involves a combination of a CNN model and an XGBoost model, a +customized model aggregation that can work with this model combination, incorporate the +usage of an early stopping mechanism to stop training when needed and incorporate the +usage of wandb for fine-tuning purposes. +""" + +# Flower server + +import timeit +from logging import DEBUG, INFO +from typing import Dict, List, Optional, Tuple, Union + +import flwr as fl +import wandb +from flwr.common import EvaluateRes, FitRes, Parameters, Scalar, parameters_to_ndarrays +from flwr.common.logger import log +from flwr.common.typing import GetParametersIns +from flwr.server.client_manager import ClientManager +from flwr.server.client_proxy import ClientProxy +from flwr.server.history import History +from flwr.server.server import evaluate_clients, fit_clients +from flwr.server.strategy import Strategy +from omegaconf import DictConfig +from torch.utils.data import DataLoader +from xgboost import XGBClassifier, XGBRegressor + +from hfedxgboost.models import CNN +from hfedxgboost.utils import EarlyStop, single_tree_preds_from_each_client, test + +FitResultsAndFailures = Tuple[ + List[Tuple[ClientProxy, FitRes]], + List[Union[Tuple[ClientProxy, FitRes], BaseException]], +] +EvaluateResultsAndFailures = Tuple[ + List[Tuple[ClientProxy, EvaluateRes]], + List[Union[Tuple[ClientProxy, EvaluateRes], BaseException]], +] + + +class FlServer(fl.server.Server): + """The FL_Server class is a sub-class of the fl.server.Server class. + + Attributes + ---------- + client_manager (ClientManager):responsible for managing the clients. + parameters (Parameters): The model parameters used for training + and evaluation. + strategy (Strategy): The strategy used for selecting clients + and aggregating results. + max_workers (None or int): The maximum number of workers + for parallel execution. + early_stopper (EarlyStop): The early stopper used for + determining when to stop training. + + Methods + ------- + fit_round(server_round, timeout): + Runs a round of fitting on the server side. + check_res_cen(current_round, timeout, start_time, history): + Gets results after fitting the model for the current round + and checks if the training should stop. + fit(num_rounds, timeout): + Runs federated learning for a given number of rounds. + evaluate_round(server_round, timeout): + Validates the current global model on a number of clients. + _get_initial_parameters(timeout): + Gets initial parameters from one of the available clients. + serverside_eval(server_round, parameters, config, + cfg, testloader, batch_size): + Performs server-side evaluation. + """ + + def __init__( + self, + cfg: DictConfig, + client_manager: ClientManager, + early_stopper: EarlyStop, + strategy: Strategy, + ) -> None: + super().__init__(client_manager=client_manager) + self.cfg = cfg + self._client_manager = client_manager + self.parameters = Parameters(tensors=[], tensor_type="numpy.ndarray") + self.strategy = strategy + self.early_stopper = early_stopper + + def fit_round( + self, + server_round: int, + timeout: Optional[float], + ): + """Run a round of fitting on the server side. + + Parameters + ---------- + self: The instance of the class. + server_round (int): The round of server communication. + timeout (float, optional): Maximum time to wait for client responses. + + Returns + ------- + The aggregated CNN model and tree + Aggregated metric value + A tuple containing the results and failures. + + None if no clients were selected. + """ + # Get clients and their respective instructions from strategy + client_instructions = self.strategy.configure_fit( + server_round=server_round, + parameters=self.parameters, + client_manager=self._client_manager, + ) + + if not client_instructions: + log(INFO, "fit_round %s: no clients selected, cancel", server_round) + return None + log( + DEBUG, + "fit_round %s: strategy sampled %s clients (out of %s)", + server_round, + len(client_instructions), + self._client_manager.num_available(), + ) + # Collect `fit` results from all clients participating in this round + if self.cfg.server.max_workers == "None": + max_workers = None + else: + max_workers = int(self.cfg.server.max_workers) + results, failures = fit_clients( + client_instructions=client_instructions, + max_workers=max_workers, + timeout=timeout, + ) + + log( + DEBUG, + "fit_round %s received %s results and %s failures", + server_round, + len(results), + len(failures), + ) + + # metrics_aggregated: Dict[str, Scalar] + aggregated_parm, metrics_aggregated = self.strategy.aggregate_fit( + server_round, results, failures + ) + # the tests is convinced that aggregated_parm is a Parameters | None + # which is not true as aggregated_parm is actually List[Union[Parameters,None]] + if aggregated_parm: + cnn_aggregated, trees_aggregated = aggregated_parm[0], aggregated_parm[1] # type: ignore # noqa: E501 # pylint: disable=line-too-long + else: + raise Exception("aggregated parameters is None") + + if isinstance(trees_aggregated, list): + print("Server side aggregated", len(trees_aggregated), "trees.") + else: + print("Server side did not aggregate trees.") + + return ( + (cnn_aggregated, trees_aggregated), + metrics_aggregated, + (results, failures), + ) + + def check_res_cen(self, current_round, timeout, start_time, history): + """Get results after fitting the model for the current round. + + Check if those results are not None and check + if the training should stop or not. + + Parameters + ---------- + current_round (int): The current round number. + timeout (int): The time limit for the evaluation. + start_time (float): The starting time of the evaluation. + history (History): The object for storing the evaluation history. + + Returns + ------- + bool: True if the early stop criteria is met, False otherwise. + """ + res_fit = self.fit_round(server_round=current_round, timeout=timeout) + if res_fit: + parameters_prime, _, _ = res_fit + if parameters_prime: + self.parameters = parameters_prime + res_cen = self.strategy.evaluate(current_round, parameters=self.parameters) + if res_cen is not None: + loss_cen, metrics_cen = res_cen + log( + INFO, + "fit progress: (%s, %s, %s, %s)", + current_round, + loss_cen, + metrics_cen, + timeit.default_timer() - start_time, + ) + history.add_loss_centralized(server_round=current_round, loss=loss_cen) + history.add_metrics_centralized( + server_round=current_round, metrics=metrics_cen + ) + if self.cfg.use_wandb: + wandb.log({"server_metric_value": metrics_cen, "server_loss": loss_cen}) + if self.early_stopper.early_stop(res_cen): + return True + return False + + # pylint: disable=too-many-locals + def fit(self, num_rounds: int, timeout: Optional[float]) -> History: + """Run federated learning for a given number of rounds. + + Parameters + ---------- + num_rounds (int): The number of rounds to run federated learning. + timeout (Optional[float]): The optional timeout value for each round. + + Returns + ------- + History: The history object that stores the loss and metrics data. + """ + history = History() + + # Initialize parameters + log(INFO, "Initializing global parameters") + self.parameters = self._get_initial_parameters(timeout=timeout) + + log(INFO, "Evaluating initial parameters") + res = self.strategy.evaluate(0, parameters=self.parameters) + if res is not None: + log( + INFO, + "initial parameters (loss, other metrics): %s, %s", + res[0], + res[1], + ) + history.add_loss_centralized(server_round=0, loss=res[0]) + history.add_metrics_centralized(server_round=0, metrics=res[1]) + + # Run federated learning for num_rounds + log(INFO, "FL starting") + start_time = timeit.default_timer() + for current_round in range(1, num_rounds + 1): + stop = self.check_res_cen(current_round, timeout, start_time, history) + # Evaluate model on a sample of available clients + res_fed = self.evaluate_round(server_round=current_round, timeout=timeout) + if res_fed: + loss_fed, evaluate_metrics_fed, _ = res_fed + if loss_fed: + history.add_loss_distributed( + server_round=current_round, loss=loss_fed + ) + history.add_metrics_distributed( + server_round=current_round, metrics=evaluate_metrics_fed + ) + # Stop if no progress is happening + if stop: + break + + # Bookkeeping + end_time = timeit.default_timer() + elapsed = end_time - start_time + log(INFO, "FL finished in %s", elapsed) + return history + + def evaluate_round( + self, + server_round: int, + timeout: Optional[float], + ) -> Optional[ + Tuple[Optional[float], Dict[str, Scalar], EvaluateResultsAndFailures] + ]: + """Validate current global model on a number of clients. + + Parameters + ---------- + server_round (int): representing the current server round + timeout (float, optional): The time limit for the request in seconds. + + Returns + ------- + Aggregated loss, + Aggregated metric, + Tuple of the results and failures. + or + None if no clients selected. + """ + # Get clients and their respective instructions from strategy + client_instructions = self.strategy.configure_evaluate( + server_round=server_round, + parameters=self.parameters, + client_manager=self._client_manager, + ) + if not client_instructions: + log(INFO, "evaluate_round %s: no clients selected, cancel", server_round) + return None + log( + DEBUG, + "evaluate_round %s: strategy sampled %s clients (out of %s)", + server_round, + len(client_instructions), + self._client_manager.num_available(), + ) + + # Collect `evaluate` results from all clients participating in this round + results, failures = evaluate_clients( + client_instructions, + max_workers=self.max_workers, + timeout=timeout, + ) + log( + DEBUG, + "evaluate_round %s received %s results and %s failures", + server_round, + len(results), + len(failures), + ) + + # Aggregate the evaluation results + aggregated_result = self.strategy.aggregate_evaluate( + server_round, results, failures + ) + + loss_aggregated, metrics_aggregated = aggregated_result + return loss_aggregated, metrics_aggregated, (results, failures) + + def _get_initial_parameters(self, timeout: Optional[float]): + """Get initial parameters from one of the available clients. + + Parameters + ---------- + timeout (float, optional): The time limit for the request in seconds. + Defaults to None. + + Returns + ------- + parameters (tuple): A tuple containing the initial parameters. + """ + log(INFO, "Requesting initial parameters from one random client") + random_client = self._client_manager.sample(1)[0] + ins = GetParametersIns(config={}) + get_parameters_res_tree = random_client.get_parameters(ins=ins, timeout=timeout) + log(INFO, "Received initial parameters from one random client") + + return (get_parameters_res_tree[0].parameters, get_parameters_res_tree[1]) # type: ignore # noqa: E501 # pylint: disable=line-too-long + + +def serverside_eval( + server_round: int, + parameters: Tuple[ + Parameters, + Union[ + Tuple[XGBClassifier, int], + Tuple[XGBRegressor, int], + List[Union[Tuple[XGBClassifier, int], Tuple[XGBRegressor, int]]], + ], + ], + config: Dict[str, Scalar], + cfg: DictConfig, + testloader: DataLoader, +) -> Tuple[float, Dict[str, float]]: + """Perform server-side evaluation. + + Parameters + ---------- + server_round (int): The round of server evaluation. + parameters (Tuple): A tuple containing the parameters needed for evaluation. + First element: an instance of the Parameters class. + Second element: a tuple consists of either an XGBClassifier + or XGBRegressor model and an integer, or a list of that tuple. + config (Dict): A dictionary containing configuration parameters. + cfg: Hydra configuration object. + testloader (DataLoader): The data loader used for testing. + batch_size (int): The batch size used for testing. + + Returns + ------- + Tuple[float, Dict]: A tuple containing the evaluation loss (float) and + a dictionary containing the evaluation metric(s) (float). + """ + print(config, server_round) + # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + metric_name = cfg.dataset.task.metric.name + + device = cfg.server.device + model = CNN(cfg) + model.set_weights(parameters_to_ndarrays(parameters[0])) + model.to(device) + + trees_aggregated = parameters[1] + testloader = single_tree_preds_from_each_client( + testloader, + cfg.run_experiment.batch_size, + trees_aggregated, + cfg.n_estimators_client, + cfg.client_num, + ) + loss, result, _ = test(cfg, model, testloader, device=device, log_progress=False) + + print( + f"Evaluation on the server: test_loss={loss:.4f},", + f"test_,{metric_name},={result:.4f}", + ) + return loss, {metric_name: result} diff --git a/baselines/hfedxgboost/hfedxgboost/strategy.py b/baselines/hfedxgboost/hfedxgboost/strategy.py new file mode 100644 index 000000000000..17436c401c30 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/strategy.py @@ -0,0 +1,5 @@ +"""Optionally define a custom strategy. + +Needed only when the strategy is not yet implemented in Flower or because you want to +extend or modify the functionality of an existing strategy. +""" diff --git a/baselines/hfedxgboost/hfedxgboost/sweep.yaml b/baselines/hfedxgboost/hfedxgboost/sweep.yaml new file mode 100644 index 000000000000..cd2ca4b118b7 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/sweep.yaml @@ -0,0 +1,28 @@ +program: main.py +method: random +#metric part is just here for documentation +#not needed as this is random strategy +metric: + goal: minimize + name: server_loss +parameters: + use_wandb: + value: True + dataset: + value: [cpusmall] + clients: + value: cpusmall_5_clients + clients.n_estimators_client: + values: [10,25,50,100,150,200,250] + clients.num_iterations: + values: [100,500] + clients.xgb.max_depth: + values: [4,5,6,7,8] + clients.CNN.lr: + min: .00001 + max: 0.001 +command: + - ${env} + - python + - ${program} + - ${args_no_hyphens} \ No newline at end of file diff --git a/baselines/hfedxgboost/hfedxgboost/utils.py b/baselines/hfedxgboost/hfedxgboost/utils.py new file mode 100644 index 000000000000..67450f7b8af7 --- /dev/null +++ b/baselines/hfedxgboost/hfedxgboost/utils.py @@ -0,0 +1,566 @@ +"""Define any utility function. + +They are not directly relevant to the other (more FL specific) python modules. For +example, you may define here things like: loading a model from a checkpoint, saving +results, plotting. +""" +import csv +import math +import os +import os.path +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from flwr.common import NDArray +from hydra.utils import instantiate +from omegaconf import DictConfig +from sklearn.metrics import accuracy_score, mean_squared_error +from torch.utils.data import DataLoader, Dataset, TensorDataset +from tqdm import tqdm +from xgboost import XGBClassifier, XGBRegressor + +from hfedxgboost.dataset import load_single_dataset +from hfedxgboost.models import CNN, fit_xgboost + +dataset_tasks = { + "a9a": "BINARY", + "cod-rna": "BINARY", + "ijcnn1": "BINARY", + "abalone": "REG", + "cpusmall": "REG", + "space_ga": "REG", + "YearPredictionMSD": "REG", +} + + +def get_dataloader( + dataset: Dataset, partition: str, batch_size: Union[int, str] +) -> DataLoader: + """Create a DataLoader object for the given dataset. + + Parameters + ---------- + dataset (Dataset): The dataset object containing the data to be loaded. + partition (str): The partition of the dataset to load. + batch_size (Union[int, str]): The size of each mini-batch. + If "whole" is specified, the entire dataset will be included in a single batch. + + Returns + ------- + loader (DataLoader): The DataLoader object. + """ + if batch_size == "whole": + batch_size = len(dataset) + return DataLoader( + dataset, batch_size=batch_size, pin_memory=True, shuffle=(partition == "train") + ) + + +def evaluate(task_type, y, preds) -> float: + """Evaluate the performance of a given model prediction based on the task type. + + Parameters + ---------- + task_type: A string representing the type of the task + (either "BINARY" or "REG"). + y: The true target values. + preds: The predicted target values. + + Returns + ------- + result: The evaluation result based on the task type. If task_type is "BINARY", + it computes the accuracy score. + If task_type is "REG", it calculates the mean squared error. + """ + if task_type.upper() == "BINARY": + result = accuracy_score(y, preds) + elif task_type.upper() == "REG": + result = mean_squared_error(y, preds) + return result + + +def run_single_exp( + config, dataset_name, task_type, n_estimators +) -> Tuple[float, float]: + """Run a single experiment using XGBoost on a dataset. + + Parameters + ---------- + - config (object): Hydra Configuration object containing necessary settings. + - dataset_name (str): Name of the dataset to train and test on. + - task_type (str): Type of the task "BINARY" or "REG". + - n_estimators (int): Number of estimators (trees) to use in the XGBoost model. + + Returns + ------- + - result_train (float): Evaluation result on the training set. + - result_test (float): Evaluation result on the test set. + """ + x_train, y_train, x_test, y_test = load_single_dataset( + task_type, dataset_name, train_ratio=config.dataset.train_ratio + ) + tree = fit_xgboost(config, task_type, x_train, y_train, n_estimators) + preds_train = tree.predict(x_train) + result_train = evaluate(task_type, y_train, preds_train) + preds_test = tree.predict(x_test) + result_test = evaluate(task_type, y_test, preds_test) + return result_train, result_test + + +def run_centralized( + config: DictConfig, dataset_name: str = "all", task_type: Optional[str] = None +) -> Union[Tuple[float, float], List[None]]: + """Run the centralized training and testing process. + + Parameters + ---------- + config (DictConfig): Hydra configuration object. + dataset_name (str): Name of the dataset to run the experiment on. + task_type (str): Type of task. + + Returns + ------- + None: Returns None if dataset_name is "all". + Tuple: Returns a tuple (result_train, result_test) of training and testing + results if dataset_name is not "all" and task_type is specified. + + Raises + ------ + Exception: Raises an exception if task_type is not specified correctly + and the dataset_name is not in the dataset_tasks dict. + """ + if dataset_name == "all": + for dataset in dataset_tasks: + result_train, result_test = run_single_exp( + config, dataset, dataset_tasks[dataset], config.n_estimators + ) + print( + "Results for", + dataset, + ", Task:", + dataset_tasks[dataset], + ", Train:", + result_train, + ", Test:", + result_test, + ) + return [] + + if task_type: + result_train, result_test = run_single_exp( + config, + dataset_name, + task_type, + config.xgboost_params_centralized.n_estimators, + ) + print( + "Results for", + dataset_name, + ", Task:", + task_type, + ", Train:", + result_train, + ", Test:", + result_test, + ) + return result_train, result_test + + if dataset_name in dataset_tasks.keys(): + result_train, result_test = run_single_exp( + config, + dataset_name, + dataset_tasks[dataset_name], + config.xgboost_params_centralized.n_estimators, + ) + print( + "Results for", + dataset_name, + ", Task:", + dataset_tasks[dataset_name], + ", Train:", + result_train, + ", Test:", + result_test, + ) + return result_train, result_test + + raise Exception( + "task_type should be assigned to be BINARY for" + "binary classification" + "tasks or REG for regression tasks" + "or the dataset should be one of the follwing" + "a9a, cod-rna, ijcnn1, space_ga, abalone,", + "cpusmall, YearPredictionMSD", + ) + + +def local_clients_performance( + config: DictConfig, trainloaders, x_test, y_test, task_type: str +) -> None: + """Evaluate the performance of clients on local data using XGBoost. + + Parameters + ---------- + config (DictConfig): Hydra configuration object. + trainloaders: List of data loaders for each client. + x_test: Test features. + y_test: Test labels. + task_type (str): Type of prediction task. + """ + for i, trainloader in enumerate(trainloaders): + for local_dataset in trainloader: + local_x_train, local_y_train = local_dataset[0], local_dataset[1] + tree = fit_xgboost( + config, + task_type, + local_x_train, + local_y_train, + 500 // config.client_num, + ) + + preds_train = tree.predict(local_x_train) + result_train = evaluate(task_type, local_y_train, preds_train) + + preds_test = tree.predict(x_test) + result_test = evaluate(task_type, y_test, preds_test) + print("Local Client %d XGBoost Training Results: %f" % (i, result_train)) + print("Local Client %d XGBoost Testing Results: %f" % (i, result_test)) + + +def single_tree_prediction( + tree, + n_tree: int, + dataset: NDArray, +) -> Optional[NDArray]: + """Perform a single tree prediction using the provided tree object on given dataset. + + Parameters + ---------- + tree (either XGBClassifier or XGBRegressor): The tree object + used for prediction. + n_tree (int): The index of the tree to be used for prediction. + dataset (NDArray): The dataset for which the prediction is to be made. + + Returns + ------- + NDArray object: representing the prediction result. + None: If the provided n_tree is larger than the total number of trees + in the tree object, and a warning message is printed. + """ + num_t = len(tree.get_booster().get_dump()) + if n_tree > num_t: + print( + "The tree index to be extracted is larger than the total number of trees." + ) + return None + + return tree.predict( + dataset, iteration_range=(n_tree, n_tree + 1), output_margin=True + ) + + +def single_tree_preds_from_each_client( + trainloader: DataLoader, + batch_size, + client_tree_ensamples: Union[ + Tuple[XGBClassifier, int], + Tuple[XGBRegressor, int], + List[Union[Tuple[XGBClassifier, int], Tuple[XGBRegressor, int]]], + ], + n_estimators_client: int, + client_num: int, +) -> Optional[Tuple[NDArray, NDArray]]: + """Predict using trees from client tree ensamples. + + Extract each tree from each tree ensample from each client, + and predict the output of the data using that tree, + place those predictions in the preds_from_all_trees_from_all_clients, + and return it. + + Parameters + ---------- + trainloader: + - a dataloader that contains the dataset to be predicted. + client_tree_ensamples: + - the trained XGBoost tree ensample from each client, + each tree ensembles comes attached + to its client id in a tuple + - can come as a single tuple of XGBoost tree ensample and + its client id or multiple tuples in one list. + + Returns + ------- + loader (DataLoader): The DataLoader object that contains the + predictions of the tree + """ + if trainloader is None: + return None + + for local_dataset in trainloader: + x_train, y_train = local_dataset[0], np.float32(local_dataset[1]) + + preds_from_all_trees_from_all_clients = np.zeros( + (x_train.shape[0], client_num * n_estimators_client), dtype=np.float32 + ) + if isinstance(client_tree_ensamples, list) is False: + temp_trees = [client_tree_ensamples[0]] * client_num + elif isinstance(client_tree_ensamples, list): + client_tree_ensamples.sort(key=lambda x: x[1]) + temp_trees = [i[0] for i in client_tree_ensamples] + if len(client_tree_ensamples) != client_num: + temp_trees += [client_tree_ensamples[0][0]] * ( + client_num - len(client_tree_ensamples) + ) + + for i, _ in enumerate(temp_trees): + for j in range(n_estimators_client): + preds_from_all_trees_from_all_clients[ + :, i * n_estimators_client + j + ] = single_tree_prediction(temp_trees[i], j, x_train) + + preds_from_all_trees_from_all_clients = torch.from_numpy( + np.expand_dims(preds_from_all_trees_from_all_clients, axis=1) + ) + y_train = torch.from_numpy(np.expand_dims(y_train, axis=-1)) + tree_dataset = TensorDataset(preds_from_all_trees_from_all_clients, y_train) + return get_dataloader(tree_dataset, "tree", batch_size) + + +def test( + cfg, + net: CNN, + testloader: DataLoader, + device: torch.device, + log_progress: bool = True, +) -> Tuple[float, float, int]: + """Evaluates the performance of a CNN model on a given test dataset. + + Parameters + ---------- + cfg (Any): The configuration object. + net (CNN): The CNN model to test. + testloader (DataLoader): The data loader for the test dataset. + device (torch.device): The device to run the evaluation on. + log_progress (bool, optional): Whether to log the progress during evaluation. + Default is True. + + Returns + ------- + Tuple[float, float, int]: A tuple containing the average loss, + average metric result, and total number of samples evaluated. + """ + criterion = instantiate(cfg.dataset.task.criterion) + metric_fn = instantiate(cfg.dataset.task.metric.fn) + + total_loss, total_result, n_samples = 0.0, 0.0, 0 + net.eval() + with torch.no_grad(): + # progress_bar = tqdm(testloader, desc="TEST") if log_progress else testloader + for data in tqdm(testloader, desc="TEST") if log_progress else testloader: + tree_outputs, labels = data[0].to(device), data[1].to(device) + outputs = net(tree_outputs) + total_loss += criterion(outputs, labels).item() + n_samples += labels.size(0) + metric_val = metric_fn(outputs.cpu(), labels.type(torch.int).cpu()) + total_result += metric_val * labels.size(0) + + if log_progress: + print("\n") + + return total_loss / n_samples, total_result / n_samples, n_samples + + +class EarlyStop: + """Stop the tain when no progress is happening.""" + + def __init__(self, cfg): + self.num_waiting_rounds = cfg.dataset.early_stop_patience_rounds + self.counter = 0 + self.min_loss = float("inf") + self.metric_value = None + + def early_stop(self, res) -> Optional[Tuple[float, float]]: + """Check if the model made any progress in number of rounds. + + If it didn't it will return the best result and the server + will stop running the fit function, if + it did it will return None, and won't stop the server. + + Parameters + ---------- + res: tuple of 2 elements, res[0] is a float that indicate the loss, + res[1] is actually a 1 element dictionary that looks like this + {'Accuracy': tensor(0.8405)} + + Returns + ------- + Optional[Tuple[float,float]]: (best loss the model achieved, + best metric value associated with that loss) + """ + loss = res[0] + metric_val = list(res[1].values())[0].item() + if loss < self.min_loss: + self.min_loss = loss + self.metric_value = metric_val + self.counter = 0 + print( + "New best loss value achieved,", + "loss", + self.min_loss, + "metric value", + self.metric_value, + ) + elif loss > (self.min_loss): + self.counter += 1 + if self.counter >= self.num_waiting_rounds: + print( + "That training is been stopped as the", + "model achieve no progress with", + "loss =", + self.min_loss, + "result =", + self.metric_value, + ) + return (self.metric_value, self.min_loss) + return None + + +# results + + +def create_res_csv(filename, fields) -> None: + """Create a CSV file with the provided file name.""" + if not os.path.isfile(filename): + with open(filename, "w") as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(fields) + + +class ResultsWriter: + """Write the results for the federated experiments.""" + + def __init__(self, cfg) -> None: + self.cfg = cfg + self.tas_type = cfg.dataset.task.task_type + if self.tas_type == "REG": + self.best_res = math.inf + self.compare_fn = min + if self.tas_type == "BINARY": + self.best_res = -1 + self.compare_fn = max + self.best_res_round_num = 0 + self.fields = [ + "dataset_name", + "client_num", + "n_estimators_client", + "num_rounds", + "xgb_max_depth", + "cnn_lr", + "best_res", + "best_res_round_num", + "num_iterations", + ] + + def extract_best_res(self, history) -> Tuple[float, int]: + """Take the history & returns the best result and its corresponding round num. + + Parameters + ---------- + history: a history object that contains metrics_centralized keys + + Returns + ------- + Tuple[float, int]: a tuple containing the best result (float) and + its corresponding round number (int) + """ + for key in history.metrics_centralized.keys(): + for i in history.metrics_centralized[key]: + if ( + self.compare_fn(i[1].item(), self.best_res) == i[1] + and i[1].item() != self.best_res + ): + self.best_res = i[1].item() + self.best_res_round_num = i[0] + return (self.best_res, self.best_res_round_num) + + def write_res(self, filename) -> None: + """Write the results of the federated model to a CSV file. + + The function opens the specified file in 'a' (append) mode and creates a + csvwriter object and add the dataset name, xgboost model's and CNN model's + hyper-parameters used, and the result. + + Parameters + ---------- + filename: string that indicates the CSV file that will be written in. + """ + row = [ + str(self.cfg.dataset.dataset_name), + str(self.cfg.client_num), + str(self.cfg.clients.n_estimators_client), + str(self.cfg.run_experiment.num_rounds), + str(self.cfg.clients.xgb.max_depth), + str(self.cfg.clients.CNN.lr), + str(self.best_res), + str(self.best_res_round_num), + str(self.cfg.run_experiment.fit_config.num_iterations), + ] + with open(filename, "a") as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(row) + + +class CentralizedResultsWriter: + """Write the results for the centralized experiments.""" + + def __init__(self, cfg) -> None: + self.cfg = cfg + self.tas_type = cfg.dataset.task.task_type + self.fields = [ + "dataset_name", + "n_estimators_client", + "xgb_max_depth", + "subsample", + "learning_rate", + "colsample_bylevel", + "colsample_bynode", + "colsample_bytree", + "alpha", + "gamma", + "num_parallel_tree", + "min_child_weight", + "result_train", + "result_test", + ] + + def write_res(self, filename, result_train, result_test) -> None: + """Write the results of the centralized model to a CSV file. + + The function opens the specified file in 'a' (append) mode and creates a + csvwriter object and add the dataset name, xgboost's + hyper-parameters used, and the result. + + Parameters + ---------- + filename: string that indicates the CSV file that will be written in. + """ + row = [ + str(self.cfg.dataset.dataset_name), + str(self.cfg.xgboost_params_centralized.n_estimators), + str(self.cfg.xgboost_params_centralized.max_depth), + str(self.cfg.xgboost_params_centralized.subsample), + str(self.cfg.xgboost_params_centralized.learning_rate), + str(self.cfg.xgboost_params_centralized.colsample_bylevel), + str(self.cfg.xgboost_params_centralized.colsample_bynode), + str(self.cfg.xgboost_params_centralized.colsample_bytree), + str(self.cfg.xgboost_params_centralized.alpha), + str(self.cfg.xgboost_params_centralized.gamma), + str(self.cfg.xgboost_params_centralized.num_parallel_tree), + str(self.cfg.xgboost_params_centralized.min_child_weight), + str(result_train), + str(result_test), + ] + with open(filename, "a") as csvfile: + csvwriter = csv.writer(csvfile) + csvwriter.writerow(row) diff --git a/baselines/hfedxgboost/pyproject.toml b/baselines/hfedxgboost/pyproject.toml new file mode 100644 index 000000000000..1fbbb85ac36b --- /dev/null +++ b/baselines/hfedxgboost/pyproject.toml @@ -0,0 +1,145 @@ +[build-system] +requires = ["poetry-core>=1.4.0"] +build-backend = "poetry.masonry.api" + +[tool.poetry] +name = "hfedxgboost" # <----- Ensure it matches the name of your baseline directory containing all the source code +version = "1.0.0" +description = "HFedXgboost: Gradient-less Federated Gradient Boosting Trees with Learnable Learning Rates" +license = "Apache-2.0" +authors = ["The Flower Authors ", "Aml Hassan Esmil "] +readme = "README.md" +homepage = "https://flower.dev" +repository = "https://github.com/adap/flower" +documentation = "https://flower.dev" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: MacOS :: MacOS X", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] + +[tool.poetry.dependencies] +python = ">=3.10.0, <3.11.0" # don't change this +flwr = { extras = ["simulation"], version = "1.5.0" } +hydra-core = "1.3.2" # don't change this +torch = "1.13.1" +scikit-learn = "1.3.0" +xgboost = "2.0.0" +torchmetrics = "1.1.2" +tqdm = "4.66.1" +torchvision = "0.14.1" +wandb = "0.15.12" + +[tool.poetry.dev-dependencies] +isort = "==5.11.5" +black = "==23.1.0" +docformatter = "==1.5.1" +mypy = "==1.4.1" +pylint = "==2.8.2" +flake8 = "==3.9.2" +pytest = "==6.2.4" +pytest-watch = "==4.2.0" +ruff = "==0.0.272" +types-requests = "==2.27.7" +virtualenv = "^20.21.0" + +[tool.isort] +line_length = 88 +indent = " " +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true + +[tool.black] +line-length = 88 +target-version = ["py38", "py39", "py310", "py311"] + +[tool.pytest.ini_options] +minversion = "6.2" +addopts = "-qq" +testpaths = [ + "flwr_baselines", +] + +[tool.mypy] +ignore_missing_imports = true +strict = false +plugins = "numpy.typing.mypy_plugin" + +[tool.pylint."MESSAGES CONTROL"] +disable = "bad-continuation,duplicate-code,too-few-public-methods,useless-import-alias" +good-names = "i,j,k,_,x,y,X,Y" +signature-mutators="hydra.main.main" + +[tool.pylint.typecheck] +generated-members="numpy.*, torch.*, tensorflow.*" + +[[tool.mypy.overrides]] +module = [ + "importlib.metadata.*", + "importlib_metadata.*", +] +follow_imports = "skip" +follow_imports_for_stubs = true +disallow_untyped_calls = false + +[[tool.mypy.overrides]] +module = "torch.*" +follow_imports = "skip" +follow_imports_for_stubs = true + +[tool.docformatter] +wrap-summaries = 88 +wrap-descriptions = 88 + +[tool.ruff] +target-version = "py38" +line-length = 88 +select = ["D", "E", "F", "W", "B", "ISC", "C4"] +fixable = ["D", "E", "F", "W", "B", "ISC", "C4"] +ignore = ["B024", "B027"] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "venv", + "proto", +] + +[tool.ruff.pydocstyle] +convention = "numpy" diff --git a/baselines/hfedxgboost/results.csv b/baselines/hfedxgboost/results.csv new file mode 100644 index 000000000000..678421bec48e --- /dev/null +++ b/baselines/hfedxgboost/results.csv @@ -0,0 +1,155 @@ +dataset_name,client_num,n_estimators_client,num_rounds,xgb_max_depth,CNN_lr,best_res,best_res_round_num,num_iterations +a9a,2,250,30,8,0.0001,0.8429285287857056,30,100 +a9a,2,250,40,8,0.0001,0.8376873135566711,40,100 +a9a,2,250,20,8,0.0001,0.7641471028327942,20,100 +a9a,2,500,30,8,0.001,0.8455491065979004,29,100 +ijcnn1,2,250,30,8,0.0005,0.9234752058982849,30,100 +ijcnn1,2,300,50,5,0.0005,0.9468683004379272,49,100 +cod-rna,10,100,30,8,0.001,0.9503719806671143,30,100 +ijcnn1,10,100,30,8,0.0005,0.938660204410553,28,100 +ijcnn1,10,100,30,8,0.0005,0.9379972219467163,28,100 +a9a,10,100,30,8,0.001,0.8403897881507874,26,100 +cod-rna,2,250,30,8,0.0001,0.9640399813652039,30,100 +cod-rna,5,100,30,8,0.001,0.9620075225830078,29,100 +abalone,5,100,30,8,0.0001,10.37040901184082,29,100 +abalone,5,100,30,4,0.0001,10.158971786499023,28,100 +abalone,5,100,30,4,0.001,10.370624542236328,24,100 +abalone,5,50,30,2,0.01,10.314363479614258,27,100 +abalone,5,500,30,2,0.01,59.639190673828125,30,100 +abalone,5,500,50,8,0.0001,10.406515121459961,39,100 +abalone,5,100,50,4,0.0001,10.35323715209961,49,100 +abalone,5,100,50,4,0.0001,10.39498233795166,49,100 +ijcnn1,2,250,50,8,0.0001,0.9802058339118958,45,500 +a9a,2,250,30,8,0.0001,0.8449758291244507,23,500 +ijcnn1,5,100,30,8,0.0001,0.9731026887893677,16,500 +ijcnn1,10,100,30,8,0.0005,0.967893660068512,23,500 +abalone,10,100,30,8,0.0005,10.410109519958496,29,500 +cpusmall,2,250,50,8,0.0001,292.26702880859375,49,500 +cpusmall,2,250,2,4,0.0001,321.2448425292969,1,500 +cpusmall,2,100,50,4,0.0001,263.4275207519531,50,500 +cpusmall,2,100,150,4,0.0001,93.43792724609375,149,500 +cpusmall,2,100,1000,4,0.0001,14.8245849609375,810,500 +space_ga,2,100,50,4,1e-05,0.025414548814296722,4,500 +space_ga,5,100,40,4,1e-06,0.060075871646404266,40,100 +space_ga,5,100,100,4,1e-06,0.03812913969159126,0,100 +space_ga,5,100,200,4,1e-06,0.04440382868051529,0,500 +space_ga,10,50,50,4,1e-05,0.23313003778457642,0,500 +space_ga,10,100,50,4,1e-05,0.025433368980884552,22,100 +a9a,2,250,30,8,0.0001,0.8328556418418884,26,100 +cpusmall,10,25,300,4,1e-05,289.2887268066406,299,50 +cpusmall,10,25,300,4,0.0001,171.87559509277344,300,100 +cpusmall,10,25,1000,4,0.0001,21.770994186401367,994,100 +YearPredictionMSD,2,250,20,8,0.0001,119.265510559082,20,100 +cpusmall,2,100,1000,4,0.0001,15.070291519165,587,500 +cpusmall,2,100,800,8,0.0001,11.7772,679,500 +a9a,2,250,30,8,0.0001,0.8457129001617432,20,500 +a9a,2,250,20,8,0.0001,0.8449758291244507,15,500 +a9a,2,250,15,8,0.0001,0.8448120355606079,9,500 +a9a,2,250,15,8,0.0001,0.8441569209098816,15,500 +a9a,2,250,15,8,0.0001,0.8456310033798218,14,500 +a9a,2,250,15,8,0.0001,0.8412087559700012,15,500 +a9a,5,100,30,8,0.0005,0.8429285287857056,30,500 +a9a,10,100,30,8,0.001,0.8362951278686523,14,100 +a9a,10,100,30,8,0.001,0.8377692103385925,15,100 +a9a,10,100,30,8,0.001,0.837441623210907,24,500 +cod-rna,2,250,30,8,0.0001,0.965569019317627,27,500 +cod-rna,5,100,30,8,0.001,0.9636204242706299,24,500 +ijcnn1,2,250,50,8,0.0001,0.9803005456924438,45,500 +abalone,10,50,30,8,0.0001,10.225870132446289,11,100 +abalone,2,250,30,8,0.0001,10.415020942687988,30,100 +cpusmall,2,500,400,8,0.0001,99999999999.999,0,500 +cpusmall,5,100,1,4,5e-05,3044.56689453125,1,500 +cpusmall,5,100,1,4,5e-05,3168.800537109375,1,500 +abalone,2,250,30,4,0.0001,10.305878639221191,27,100 +abalone,2,150,50,8,0.0001,10.236144065856934,50,100 +abalone,2,150,50,8,0.0001,10.031058311462402,50,300 +a9a,2,250,15,8,0.0001,0.8440750241279602,13,500 +a9a,5,100,30,8,0.0005,0.8422733545303345,21,500 +a9a,10,100,30,8,0.001,0.8367046117782593,22,500 +cod-rna,2,250,30,8,0.0001,0.9654850959777832,28,500 +cod-rna,5,50,1000,8,0.0001,-1,0,500 +cod-rna,5,100,30,8,0.001,0.9605903625488281,29,100 +abalone,5,100,50,4,0.03346913907439582,10.539855003356934,17,100 +abalone,5,100,50,4,0.034568059419072865,10.53787899017334,17,100 +abalone,5,100,50,4,0.04630995211462259,10.401963233947754,3,100 +abalone,5,100,50,4,0.05173366161706037,10.5382719039917,12,100 +abalone,5,100,50,4,0.08622379813597802,10.538084983825684,19,100 +abalone,5,100,50,4,0.05839912417408451,10.541743278503418,14,100 +abalone,5,100,50,4,0.029151079702933427,10.53785514831543,20,100 +abalone,5,100,50,4,0.0006444558899121101,10.226868629455566,49,100 +abalone,5,100,50,4,0.056951124793294075,10.537845611572266,11,100 +abalone,5,100,50,4,0.05010996385142899,10.539204597473145,22,100 +abalone,5,50,50,4,0.019894781508274385,10.012669563293457,42,100 +abalone,5,100,500,4,0.0001,9.894426345825195,120,500 +abalone,5,100,500,4,0.0001,10.059885025024414,75,500 +abalone,5,100,50,4,0.0001,9.870953559875488,49,100 +abalone,5,300,50,4,0.0001,10.408512115478516,45,100 +abalone,5,200,50,4,0.0001,10.455050468444824,47,100 +abalone,5,300,50,4,0.0001,10.496016502380371,46,100 +abalone,5,300,50,4,0.0001,10.372100830078125,9,100 +abalone,5,300,50,8,0.0017554930858480753,10.486079216003418,19,500 +abalone,5,200,50,8,0.045167641025122586,10.540440559387207,13,100 +ijcnn1,2,250,50,8,0.0001,0.9805530905723572,50,500 +abalone,2,650,100,4,0.001217139626212002,10.490303993225098,10,500 +abalone,2,150,100,6,0.009503176921678468,7.928699016571045,98,500 +abalone,2,350,100,8,0.00783944676415376,9.897452354431152,79,100 +abalone,2,200,100,8,0.008207079744179863,10.53787612915039,28,100 +abalone,2,650,100,4,0.001217139626212002,10.463903427124023,23,500 +abalone,2,150,100,6,0.009503176921678468,7.670563220977783,100,500 +abalone,2,350,100,8,0.00783944676415376,10.538168907165527,29,100 +abalone,2,150,100,5,0.009531130088689884,8.326595306396484,97,500 +abalone,2,200,100,6,0.0054653156736999805,8.855677604675293,98,500 +abalone,2,50,100,5,0.0017955547171898984,6.178965091705322,96,500 +abalone,2,150,100,5,0.004387414920659405,8.143877983093262,100,500 +abalone,2,25,100,5,0.00950017009848373,9.77553939819336,27,500 +abalone,2,50,100,5,0.004882675431964606,5.8692779541015625,96,500 +abalone,2,25,100,5,0.007557058187901093,6.141476154327393,100,100 +abalone,2,10,100,7,0.005130176232078475,7.082235813140869,92,100 +abalone,2,50,100,6,0.002823108080376602,5.545757293701172,95,100 +abalone,2,10,100,6,0.0028521611812793155,7.876493453979492,98,100 +abalone,5,25,100,5,0.009785294236382034,9.208296775817871,99,500 +abalone,5,100,100,6,0.007079031827661472,10.32761001586914,1,500 +abalone,5,10,100,7,0.001344804317317952,8.094303131103516,100,500 +abalone,5,25,100,5,0.004714981058473264,9.583475112915039,96,500 +abalone,5,100,100,7,0.006981291837612061,9.868386268615723,100,100 +abalone,5,10,100,7,0.004135357042596778,9.329564094543457,98,500 +abalone,5,100,100,7,0.00362985132837509,10.195948600769043,45,500 +abalone,5,50,100,6,0.009059669650367688,10.281177520751953,7,500 +abalone,5,25,100,7,0.005573120595947365,10.041385650634766,2,500 +abalone,5,25,100,7,0.00028621587028303236,9.260563850402832,100,500 +abalone,5,25,100,8,1.5711330591445135e-05,10.278858184814453,34,500 +abalone,5,5,100,8,0.00012332785174116285,9.62313175201416,10,500 +abalone,5,25,100,4,0.0006278898956762251,8.320752143859863,97,100 +abalone,5,25,100,6,0.0003372569452123535,8.360357284545898,99,500 +abalone,5,5,100,4,0.0002038313592678424,9.912229537963867,10,100 +abalone,5,5,100,7,0.00041985945288375667,9.698519706726074,19,500 +abalone,5,25,100,6,0.0007804153506712646,7.4677205085754395,100,500 +abalone,5,5,100,4,0.0005335030582873437,9.64600944519043,5,100 +abalone,5,25,100,6,0.0009397031028496506,7.2984700202941895,100,500 +abalone,5,25,100,8,0.0002193902512638236,9.670918464660645,99,100 +abalone,5,5,100,6,0.0005419520842675534,9.692630767822266,3,100 +abalone,5,25,100,4,0.0007623175785101614,8.790078163146973,99,500 +abalone,10,10,100,6,0.0028231080803766,9.8931884765625,10,100 +abalone,10,10,100,6,0.0028231080803766,9.223672866821289,100,100 +abalone,10,10,200,6,0.0028231080803766,9.795281410217285,64,100 +abalone,5,50,200,6,0.0004549072000953885,6.875454425811768,200,500 +cpusmall,5,25,500,6,0.000457414512764587,14.813268661499023,407,100 +cpusmall,5,25,500,6,0.0003909693357440312,15.414558410644531,292,500 +cpusmall,5,25,400,6,0.000457414512764587,15.857512474060059,366,100 +cpusmall,5,25,450,6,0.000457414512764587,15.76364803314209,208,100 +cpusmall,5,25,500,6,0.000457414512764587,15.136177062988281,234,100 +space_ga,2,10,20,8,1e-05,0.025517474859952927,9,100 +space_ga,2,10,20,8,1e-05,0.02540592849254608,10,100 +space_ga,2,10,20,8,1e-05,0.06827209889888763,20,100 +space_ga,2,10,20,8,1e-05,0.025553066283464432,1,100 +space_ga,2,10,20,8,1e-05,0.02541356533765793,5,100 +space_ga,2,10,20,8,1e-05,0.2901467978954315,20,100 +space_ga,2,10,20,8,1e-05,0.04546191915869713,20,100 +space_ga,2,10,20,8,1e-05,0.23150818049907684,0,100 +space_ga,2,10,20,8,1e-05,0.025417599827051163,9,100 +space_ga,2,10,20,8,1e-05,0.33422788977622986,16,100 +space_ga,2,10,20,8,1e-05,0.05376894772052765,0,100 +space_ga,2,10,20,8,1e-05,0.026506789028644562,0,100 +space_ga,2,10,20,8,1e-05,0.044850919395685196,0,100 +space_ga,2,10,20,8,1e-05,0.025532562285661697,0,100 +space_ga,2,10,20,8,1e-05,0.04196251928806305,20,100 diff --git a/baselines/hfedxgboost/results_centralized.csv b/baselines/hfedxgboost/results_centralized.csv new file mode 100644 index 000000000000..bf276c4cd535 --- /dev/null +++ b/baselines/hfedxgboost/results_centralized.csv @@ -0,0 +1,2 @@ +dataset_name,n_estimators_client,xgb_max_depth,subsample,learning_rate,colsample_bylevel,colsample_bynode,colsample_bytree,alpha,gamma,num_parallel_tree,min_child_weight,result_train,result_test +abalone,200,3,0.4,0.05,1,1,1,5,10,1,5,3.9291864339707256,4.404329529975106 diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index 8fbfb4843756..58bb6cbf4254 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -2,6 +2,10 @@ ## Unreleased +- **Update Flower Baselines** + + - HFedXGBoost [#2226](https://github.com/adap/flower/pull/2226) + ## v1.6.0 (2023-11-28) ### Thanks to our contributors