From 5e3c907d06f43a8c78eb2b6b1c70f420fd85e57f Mon Sep 17 00:00:00 2001 From: yes Date: Sun, 17 Nov 2024 22:35:39 -0800 Subject: [PATCH] removed code related to python native API Signed-off-by: yes --- .github/workflows/pki.yml | 52 -- docs/about/features_index/pynative.rst | 17 - .../advanced_topics/overriding_agg_fn.rst | 14 +- .../overriding_plan_settings.rst | 99 --- .../running_the_federation.notebook.rst | 219 ------ .../utilities/splitters_data.rst | 15 +- docs/get_started/examples.rst | 14 - .../examples/python_native_pytorch_mnist.rst | 173 ----- docs/source/api/openfl_native.rst | 16 - ...derated_FedProx_Keras_MNIST_Tutorial.ipynb | 376 ---------- ...rated_FedProx_PyTorch_MNIST_Tutorial.ipynb | 523 ------------- .../Federated_Keras_MNIST_Tutorial.ipynb | 280 ------- .../Federated_PyTorch_TinyImageNet.ipynb | 378 ---------- .../Federated_PyTorch_UNET_Tutorial.ipynb | 545 -------------- .../Federated_Pytorch_MNIST_Tutorial.ipynb | 267 ------- ...ch_MNIST_custom_aggregation_Tutorial.ipynb | 708 ------------------ .../torch_llm_horovod/src/InHorovodrun.py | 4 +- openfl/native/__init__.py | 7 - openfl/native/fastestimator.py | 227 ------ openfl/native/native.py | 370 --------- setup.py | 1 - tests/github/pki_wrong_cn.py | 96 --- tests/github/python_native_tf.py | 140 ---- tests/github/python_native_torch.py | 97 --- tests/openfl/native/__init__.py | 3 - tests/openfl/native/base_example.yaml | 8 - tests/openfl/native/test_update_plan.py | 104 --- 27 files changed, 4 insertions(+), 4749 deletions(-) delete mode 100644 .github/workflows/pki.yml delete mode 100644 docs/about/features_index/pynative.rst delete mode 100644 docs/developer_guide/advanced_topics/overriding_plan_settings.rst delete mode 100644 docs/developer_guide/running_the_federation.notebook.rst delete mode 100644 docs/get_started/examples/python_native_pytorch_mnist.rst delete mode 100644 docs/source/api/openfl_native.rst delete mode 100644 openfl-tutorials/Federated_FedProx_Keras_MNIST_Tutorial.ipynb delete mode 100644 openfl-tutorials/Federated_FedProx_PyTorch_MNIST_Tutorial.ipynb delete mode 100644 openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb delete mode 100644 openfl-tutorials/Federated_PyTorch_TinyImageNet.ipynb delete mode 100644 openfl-tutorials/Federated_PyTorch_UNET_Tutorial.ipynb delete mode 100644 openfl-tutorials/Federated_Pytorch_MNIST_Tutorial.ipynb delete mode 100644 openfl-tutorials/Federated_Pytorch_MNIST_custom_aggregation_Tutorial.ipynb delete mode 100644 openfl/native/__init__.py delete mode 100644 openfl/native/fastestimator.py delete mode 100644 openfl/native/native.py delete mode 100644 tests/github/pki_wrong_cn.py delete mode 100644 tests/github/python_native_tf.py delete mode 100644 tests/github/python_native_torch.py delete mode 100644 tests/openfl/native/__init__.py delete mode 100644 tests/openfl/native/base_example.yaml delete mode 100644 tests/openfl/native/test_update_plan.py diff --git a/.github/workflows/pki.yml b/.github/workflows/pki.yml deleted file mode 100644 index 7d4f90df5c..0000000000 --- a/.github/workflows/pki.yml +++ /dev/null @@ -1,52 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Private Key Infrastructure - -on: - pull_request: - branches: [ develop ] - types: [opened, synchronize, reopened, ready_for_review] - -permissions: - contents: read - -env: - # A workaround for long FQDN names provided by GitHub actions. - FQDN: "localhost" - -jobs: - test_insecure_client: - if: github.event.pull_request.draft == false - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install . - - name: Test PKI - run: | - python tests/github/pki_insecure_client.py - test_wrong_common_name: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install . - - name: Test PKI - run: | - python tests/github/pki_wrong_cn.py \ No newline at end of file diff --git a/docs/about/features_index/pynative.rst b/docs/about/features_index/pynative.rst deleted file mode 100644 index 83d4143cf8..0000000000 --- a/docs/about/features_index/pynative.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. # Copyright (C) 2020-2023 Intel Corporation -.. # SPDX-License-Identifier: Apache-2.0 - -.. this page is not be included yet, so it's marked as an orphan. -.. Remove the below line when you're ready to publish this page. - -:orphan: - -================= -Python Native API -================= - -TODO - -.. toctree -.. overview.how_can_intel_protect_federated_learning -.. overview.what_is_intel_federated_learning \ No newline at end of file diff --git a/docs/developer_guide/advanced_topics/overriding_agg_fn.rst b/docs/developer_guide/advanced_topics/overriding_agg_fn.rst index ba393f86ca..230e909473 100644 --- a/docs/developer_guide/advanced_topics/overriding_agg_fn.rst +++ b/docs/developer_guide/advanced_topics/overriding_agg_fn.rst @@ -7,19 +7,7 @@ Override Aggregation Function ***************************** -With the aggregator-based workflow, you can use custom aggregation functions for each task via Python\*\ API or command line interface. - - -Python API -========== - -1. Create an implementation of :class:`openfl.interface.aggregation_functions.core.AggregationFunction`. - -2. In the ``override_config`` keyword argument of the :func:`openfl.native.run_experiment` native function, pass the implementation as a ``tasks.{task_name}.aggregation_type`` parameter. - -.. note:: - See `Federated PyTorch MNIST Tutorial `_ for an example of the custom aggregation function. - +With the aggregator-based workflow, you can use custom aggregation functions for each task via command line interface. Command Line Interface ====================== diff --git a/docs/developer_guide/advanced_topics/overriding_plan_settings.rst b/docs/developer_guide/advanced_topics/overriding_plan_settings.rst deleted file mode 100644 index 629e8a017a..0000000000 --- a/docs/developer_guide/advanced_topics/overriding_plan_settings.rst +++ /dev/null @@ -1,99 +0,0 @@ -.. # Copyright (C) 2020-2023 Intel Corporation -.. # SPDX-License-Identifier: Apache-2.0 - -.. _overriding_plan_settings: - -*********************** -Updating plan settings -*********************** - -With the director-based workflow, you can use custom plan settings before starting the experiment. Changing plan settings in command line interface is straightforward by modifying plan.yaml. -When using Python API or Director Envoy based interactive API (Deprecated), **override_config** can be used to update plan settings. - - -Python API -========== - -Modify the plan settings: - -.. code-block:: python - - final_fl_model = fx.run_experiment(collaborators, override_config={ - 'aggregator.settings.rounds_to_train': 5, - 'aggregator.settings.log_metric_callback': write_metric, - }) - - -Director Envoy Based Interactive API Interface (Deprecated) -=========================================================== -Once you create an FL_experiment object, a basic federated learning plan with default settings is created. To check the default plan settings, print the plan as shown below: - -.. code-block:: python - - fl_experiment = FLExperiment(federation=federation, experiment_name=experiment_name) - import openfl.native as fx - print(fx.get_plan(fl_plan=fl_experiment.plan)) - -Here is an example of the default plan settings that get displayed: - -.. code-block:: python - - "aggregator.settings.best_state_path": "save/best.pbuf", - "aggregator.settings.db_store_rounds": 2, - "aggregator.settings.init_state_path": "save/init.pbuf", - "aggregator.settings.last_state_path": "save/last.pbuf", - "aggregator.settings.rounds_to_train": 10, - "aggregator.settings.write_logs": true, - "aggregator.template": "openfl.component.Aggregator", - "assigner.settings.task_groups.0.name": "train_and_validate", - "assigner.settings.task_groups.0.percentage": 1.0, - "assigner.settings.task_groups.0.tasks.0": "aggregated_model_validation", - "assigner.settings.task_groups.0.tasks.1": "train", - "assigner.settings.task_groups.0.tasks.2": "locally_tuned_model_validation", - "assigner.template": "openfl.component.RandomGroupedAssigner", - "collaborator.settings.db_store_rounds": 1, - "collaborator.settings.delta_updates": false, - "collaborator.settings.opt_treatment": "RESET", - "collaborator.template": "openfl.component.Collaborator", - "compression_pipeline.settings": {}, - "compression_pipeline.template": "openfl.pipelines.NoCompressionPipeline", - "data_loader.settings": {}, - "data_loader.template": "openfl.federated.DataLoader", - "network.settings.agg_addr": "auto", - "network.settings.agg_port": "auto", - "network.settings.cert_folder": "cert", - "network.settings.client_reconnect_interval": 5, - "network.settings.disable_client_auth": false, - "network.settings.hash_salt": "auto", - "network.settings.tls": true, - "network.template": "openfl.federation.Network", - "task_runner.settings": {}, - "task_runner.template": "openfl.federated.task.task_runner.CoreTaskRunner", - "tasks.settings": {} - - -Use **override_config** with FL_experiment.start to make any changes to the default plan settings. It's essentially a dictionary with the keys corresponding to plan parameters along with the corresponding values (or list of values). Any new key entry will be added to the plan and for any existing key present in the plan, the value will be overrriden. - - -.. code-block:: python - - fl_experiment.start(model_provider=MI, - task_keeper=TI, - data_loader=fed_dataset, - rounds_to_train=5, - opt_treatment='CONTINUE_GLOBAL', - override_config={'aggregator.settings.db_store_rounds': 1, 'compression_pipeline.template': 'openfl.pipelines.KCPipeline', 'compression_pipeline.settings.n_clusters': 2}) - - -Since 'aggregator.settings.db_store_rounds' and 'compression_pipeline.template' fields are already present in the plan, the values of these fields get replaced. Field 'compression_pipeline.settings.n_clusters' is a new entry that gets added to the plan: - -.. code-block:: python - - INFO Updating aggregator.settings.db_store_rounds to 1... native.py:102 - - INFO Updating compression_pipeline.template to openfl.pipelines.KCPipeline... native.py:102 - - INFO Did not find compression_pipeline.settings.n_clusters in config. Make sure it should exist. Creating... native.py:105 - - -A full implementation can be found at `Federated_Pytorch_MNIST_Tutorial.ipynb `_ and at `Tensorflow_MNIST.ipynb `_. diff --git a/docs/developer_guide/running_the_federation.notebook.rst b/docs/developer_guide/running_the_federation.notebook.rst deleted file mode 100644 index ce8c75df72..0000000000 --- a/docs/developer_guide/running_the_federation.notebook.rst +++ /dev/null @@ -1,219 +0,0 @@ -.. # Copyright (C) 2020-2023 Intel Corporation -.. # SPDX-License-Identifier: Apache-2.0 - -.. _running_notebook: - -********************************** -Aggregator-Based Workflow Tutorial -********************************** - -You will start a Jupyter\* \ lab server and receive a URL you can use to access the tutorials. Jupyter notebooks are provided for PyTorch\* \ and TensorFlow\* \ that simulate a federation on a local machine. - -.. note:: - - Follow the procedure to become familiar with the APIs used in aggregator-based workflow and conventions such as *FL Plans*, *Aggregators*, and *Collaborators*. - - -Start the Tutorials -=================== - -1. Start a Python\* \ 3.8 (>=3.6, <3.9) virtual environment and confirm |productName| is available. - - .. code-block:: python - - fx - - You should see a list of available commands - -2. Start a Jupyter server. This returns a URL to access available tutorials. - - .. code-block:: python - - fx tutorial start - -3. Open the URL (including the token) in your browser. - -4. Choose a tutorial from which to start. Each tutorial is a demonstration of a simulated federated learning. The following are examples of available tutorials: - - - :code:`Federated Keras MNIST Tutorial`: workspace with a simple `Keras `_ CNN model that will download the `MNIST `_ dataset and train in a federation. - - :code:`Federated Pytorch MNIST Tutorial`: workspace with a simple `PyTorch `_ CNN model that will download the `MNIST `_ dataset and train in a federation. - - :code:`Federated PyTorch UNET Tutorial`: workspace with a UNET `PyTorch `_ model that will download the `Hyper-Kvasir `_ dataset and train in a federation. - - :code:`Federated PyTorch TinyImageNet`: workspace with a MobileNet-V2 `PyTorch `_ model that will download the `Tiny-ImageNet `_ dataset and train in a federation. - - -Familiarize with the API Concepts in an Aggregator-Based Worklow -================================================================ - -Step 1: Enable the |productName| Python API -------------------------------------------- - -Add the following lines to your Python script. - - .. code-block:: python - - import openfl.native as fx - from openfl.federated import FederatedModel, FederatedDataSet - -This loads the |productName| package and import wrappers that adapt your existing data and models to a (simulated) federated context. - -Step 2: Set Up the Experiment ------------------------------ - -For a basic experiment, run the following command. - - .. code-block:: python - - fx.init() - - -This creates a workspace directory containing default FL plan values for your experiments, and sets up a an experiment with two collaborators (the collaborators are creatively named **one** and **two**). - -For an experiment with more collaborators, run the following command. - - .. code-block:: python - - collaborator_list = [str(i) for i in range(NUM_COLLABORATORS)] - fx.init('keras_cnn_mnist', col_names=collaborator_list) - - -.. note:: - - The following are template recommendations for training models: - - - For Keras models, run :code:`fx.init('keras_cnn_mnist')` to start with the *keras_cnn_mnist* template. - - For PyTorch models, run :code:`fx.init('torch_cnn_mnist')` to start with the *torch_cnn_mnist* template. - - -Step 3: Customize the Federated Learning Plan (FL Plan) -------------------------------------------------------- - -For this example, the experiment is set up with the *keras_cnn_mnist* template. - - .. code-block:: python - - fx.init('keras_cnn_mnist') - - -See the FL plan values that can be set with the :code:`fx.get_plan()` command. - - .. code-block:: python - - print(fx.get_plan()) - - { - "aggregator.settings.best_state_path": "save/keras_cnn_mnist_best.pbuf", - "aggregator.settings.init_state_path": "save/keras_cnn_mnist_init.pbuf", - "aggregator.settings.last_state_path": "save/keras_cnn_mnist_last.pbuf", - "aggregator.settings.rounds_to_train": 10, - "aggregator.template": "openfl.component.Aggregator", - ... - } - -Based on this plan values, the experiment will run for 10 rounds. You can customize the experiment to run for 20 rounds either at runtime or ahead of time. - -Set the value at **runtime** with the :code:`override-config` parameter of :code:`fx.run_experiment`. - - .. code-block:: python - - #set values at experiment runtime - fx.run_experiment(experiment_collaborators, override_config={"aggregator.settings.rounds_to_train": 20}) - - -Set the value **ahead of time** with :code:`fx.update_plan()`. - - .. code-block:: python - - #Set values ahead of time with fx.update_plan() - fx.update_plan({"aggregator.settings.rounds_to_train": 20}) - - -Step 4: Wrap the Data and Model -------------------------------- - -Use the :code:`FederatedDataSet` function to wrap in-memory numpy datasets and split the data into N mutually-exclusive chunks for each collaborator participating in the experiment. - - .. code-block:: python - - fl_data = FederatedDataSet(train_images, train_labels, valid_images, valid_labels, batch_size=32, num_classes=classes) - -Similarly, the :code:`FederatedModel` function takes as an argument your model definition. For the first example, you can wrap a Keras model in a function that outputs the compiled model. - -**Example 1:** - - .. code-block:: python - - def build_model(feature_shape,classes): - #Defines the MNIST model - model = Sequential() - model.add(Dense(64, input_shape=feature_shape, activation='relu')) - model.add(Dense(64, activation='relu')) - model.add(Dense(classes, activation='softmax')) - - model.compile(optimizer='adam', loss='categorical_crossentropy',metrics=['accuracy']) - return model - - fl_model = FederatedModel(build_model, data_loader=fl_data) - -For the second example with a PyTorch model, the :code:`FederatedModel` function takes the following parameters: - -- The class that defines the network definition and associated forward function -- The lambda optimizer method that can be set to a newly instantiated network -- The loss function - -**Example 2:** - - .. code-block:: python - - class Net(nn.Module): - def __init__(self): - super(Net, self).__init__() - self.conv1 = nn.Conv2d(1, 16, 3) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(16, 32, 3) - self.fc1 = nn.Linear(32 * 5 * 5, 32) - self.fc2 = nn.Linear(32, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(x.size(0),-1) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return F.log_softmax(x, dim=1) - - optimizer = lambda x: optim.Adam(x, lr=1e-4) - - def cross_entropy(output, target): - """Binary cross-entropy metric - """ - return F.binary_cross_entropy_with_logits(input=output,target=target) - - fl_model = FederatedModel(build_model=Net, optimizer=optimizer, loss_fn=cross_entropy, data_loader=fl_data) - - -Step 5: Define the Collaborators --------------------------------- - -Define the collaborators taking part in the experiment. The example below uses the collaborator list, created earlier with the the :code:`fx.init()` command. - - .. code-block:: python - - experiment_collaborators = {col_name:col_model for col_name, col_model \ - in zip(collaborator_list, fl_model.setup(len(collaborator_list)))} - -This command creates a model for each collaborator with their data shard. - -.. note:: - - In production deployments of |productName|, each collaborator will have the data on premise. Splitting data into shards is not necessary. - -Step 6: Run the Experiment --------------------------- - -Run the experiment for five rounds and return the final model once completed. - - .. code-block:: python - - final_fl_model = fx.run_experiment(experiment_collaborators, override_config={"aggregator.settings.rounds_to_train": 5}) \ No newline at end of file diff --git a/docs/developer_guide/utilities/splitters_data.rst b/docs/developer_guide/utilities/splitters_data.rst index 56b08ee6a1..0ebb27d63f 100644 --- a/docs/developer_guide/utilities/splitters_data.rst +++ b/docs/developer_guide/utilities/splitters_data.rst @@ -13,20 +13,7 @@ Dataset Splitters You may apply data splitters differently depending on the |productName| workflow that you follow. -OPTION 1: Use **Native Python API** (Aggregator-Based Workflow) Functions to Split the Data -=========================================================================================== - -Predefined |productName| data splitters functions are as follows: - -- ``openfl.utilities.data_splitters.EqualNumPyDataSplitter`` (default) -- ``openfl.utilities.data_splitters.RandomNumPyDataSplitter`` -- ``openfl.interface.aggregation_functions.LogNormalNumPyDataSplitter``, which assumes the ``data`` argument as ``np.ndarray`` of integers (labels) -- ``openfl.interface.aggregation_functions.DirichletNumPyDataSplitter``, which assumes the ``data`` argument as ``np.ndarray`` of integers (labels) - -Alternatively, you can create an `implementation `_ of :class:`openfl.plugins.data_splitters.NumPyDataSplitter` and pass it to the :code:`FederatedDataset` function as either ``train_splitter`` or ``valid_splitter`` keyword argument. - - -OPTION 2: Use Dataset Splitters in your Shard Descriptor +OPTION 1: Use Dataset Splitters in your Shard Descriptor ======================================================== Apply one of previously mentioned splitting function on your data to perform a simulation. diff --git a/docs/get_started/examples.rst b/docs/get_started/examples.rst index f358090e06..4fbd798fc3 100644 --- a/docs/get_started/examples.rst +++ b/docs/get_started/examples.rst @@ -29,20 +29,6 @@ See :ref:`running_the_task_runner` :ref:`running_the_task_runner` -------------------------- -Python Native API -------------------------- -Intended for quick simulation purposes - -See :ref:`python_native_pytorch_mnist` - -.. toctree:: - :hidden: - :maxdepth: 1 - - examples/python_native_pytorch_mnist - - ---------------------------- Interactive API (Deprecated) ---------------------------- diff --git a/docs/get_started/examples/python_native_pytorch_mnist.rst b/docs/get_started/examples/python_native_pytorch_mnist.rst deleted file mode 100644 index 8105ad495c..0000000000 --- a/docs/get_started/examples/python_native_pytorch_mnist.rst +++ /dev/null @@ -1,173 +0,0 @@ -.. # Copyright (C) 2020-2023 Intel Corporation -.. # SPDX-License-Identifier: Apache-2.0 - -.. _python_native_pytorch_mnist: - -========================================== -Python Native API: Federated PyTorch MNIST -========================================== - -In this tutorial, we will set up a federation and train a basic PyTorch model on the MNIST dataset using the Python Native API. -See `full notebook `_. - -.. note:: - - Ensure you have installed the |productName| package. - - See :ref:`install_package` for details. - - -Install additional dependencies if not already installed - -.. code-block:: console - - $ pip install torch torchvision - -.. code-block:: python - - import numpy as np - import torch - import torch.nn as nn - import torch.nn.functional as F - import torch.optim as optim - - import torchvision - import torchvision.transforms as transforms - import openfl.native as fx - from openfl.federated import FederatedModel,FederatedDataSet - -After importing the required packages, the next step is setting up our openfl workspace. -To do this, simply run the ``fx.init()`` command as follows: - -.. code-block:: python - - #Setup default workspace, logging, etc. - fx.init('torch_cnn_mnist', log_level='METRIC', log_file='./spam_metric.log') - -Now we are ready to define our dataset and model to perform federated learning on. -The dataset should be composed of a numpy array. We start with a simple fully connected model that is trained on the MNIST dataset. - -.. code-block:: python - - def one_hot(labels, classes): - return np.eye(classes)[labels] - - transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - trainset = torchvision.datasets.MNIST(root='./data', train=True, - download=True, transform=transform) - - train_images,train_labels = trainset.train_data, np.array(trainset.train_labels) - train_images = torch.from_numpy(np.expand_dims(train_images, axis=1)).float() - - validset = torchvision.datasets.MNIST(root='./data', train=False, - download=True, transform=transform) - - valid_images,valid_labels = validset.test_data, np.array(validset.test_labels) - valid_images = torch.from_numpy(np.expand_dims(valid_images, axis=1)).float() - valid_labels = one_hot(valid_labels,10) - -.. code-block:: python - - feature_shape = train_images.shape[1] - classes = 10 - - fl_data = FederatedDataSet(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes) - - class Net(nn.Module): - def __init__(self): - super(Net, self).__init__() - self.conv1 = nn.Conv2d(1, 16, 3) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(16, 32, 3) - self.fc1 = nn.Linear(32 * 5 * 5, 32) - self.fc2 = nn.Linear(32, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(x.size(0),-1) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return F.log_softmax(x, dim=1) - - optimizer = lambda x: optim.Adam(x, lr=1e-4) - - def cross_entropy(output, target): - """Binary cross-entropy metric - """ - return F.cross_entropy(input=output,target=target) - - -Here we can define metric logging function. It should has the following signature described below. You can use it to write metrics to tensorboard or some another specific logging. - -.. code-block:: python - - from torch.utils.tensorboard import SummaryWriter - - writer = SummaryWriter('./logs/cnn_mnist', flush_secs=5) - - - def write_metric(node_name, task_name, metric_name, metric, round_number): - writer.add_scalar("{}/{}/{}".format(node_name, task_name, metric_name), - metric, round_number) - -.. code-block:: python - - #Create a federated model using the pytorch class, lambda optimizer function, and loss function - fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy,data_loader=fl_data) - -The ``FederatedModel`` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. -It provides built in federated training and validation functions that we will see used below. -Using it's setup function, collaborator models and datasets can be automatically defined for the experiment. - -.. code-block:: python - - collaborator_models = fl_model.setup(num_collaborators=2) - collaborators = {'one':collaborator_models[0],'two':collaborator_models[1]}#, 'three':collaborator_models[2]} - -.. code-block:: python - - #Original MNIST dataset - print(f'Original training data size: {len(train_images)}') - print(f'Original validation data size: {len(valid_images)}\n') - - #Collaborator one's data - print(f'Collaborator one\'s training data size: {len(collaborator_models[0].data_loader.X_train)}') - print(f'Collaborator one\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\n') - - #Collaborator two's data - print(f'Collaborator two\'s training data size: {len(collaborator_models[1].data_loader.X_train)}') - print(f'Collaborator two\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\n') - - #Collaborator three's data - #print(f'Collaborator three\'s training data size: {len(collaborator_models[2].data_loader.X_train)}') - #print(f'Collaborator three\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}') - -We can see the current plan values by running the ``fx.get_plan()`` function - -.. code-block:: python - - #Get the current values of the plan. Each of these can be overridden - print(fx.get_plan()) - -Now we are ready to run our experiment. -If we want to pass in custom plan settings, we can easily do that with the override_config parameter - -.. code-block:: python - - # Run experiment, return trained FederatedModel - - final_fl_model = fx.run_experiment(collaborators, override_config={ - 'aggregator.settings.rounds_to_train': 5, - 'aggregator.settings.log_metric_callback': write_metric, - }) - -.. code-block:: python - - #Save final model - final_fl_model.save_native('final_pytorch_model') diff --git a/docs/source/api/openfl_native.rst b/docs/source/api/openfl_native.rst deleted file mode 100644 index bd9eb608d3..0000000000 --- a/docs/source/api/openfl_native.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. # Copyright (C) 2020-2024 Intel Corporation -.. # SPDX-License-Identifier: Apache-2.0 - -************************************************* -Native Module -************************************************* - -Native modules reference: - -.. autosummary:: - :toctree: _autosummary - :template: custom-module-template.rst - :recursive: - - openfl.native - \ No newline at end of file diff --git a/openfl-tutorials/Federated_FedProx_Keras_MNIST_Tutorial.ipynb b/openfl-tutorials/Federated_FedProx_Keras_MNIST_Tutorial.ipynb deleted file mode 100644 index cc0dcc1a9c..0000000000 --- a/openfl-tutorials/Federated_FedProx_Keras_MNIST_Tutorial.ipynb +++ /dev/null @@ -1,376 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated Keras MNIST Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "#Install Tensorflow and MNIST dataset if not installed\n", - "!pip install tensorflow==2.7.0\n", - "#Alternatively you could use the intel-tensorflow build\n", - "# !pip install intel-tensorflow==2.3.0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow.keras as keras\n", - "from tensorflow.keras import backend as K\n", - "from tensorflow.keras import Sequential\n", - "from tensorflow.keras.layers import Conv2D, Flatten, Dense, MaxPool2D\n", - "from tensorflow.keras.utils import to_categorical\n", - "from tensorflow.keras.datasets import mnist\n", - "\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel,FederatedDataSet\n", - "tf.config.run_functions_eagerly(True)\n", - "tf.random.set_seed(0)\n", - "np.random.seed(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_intel_tensorflow():\n", - " \"\"\"\n", - " Check if Intel version of TensorFlow is installed\n", - " \"\"\"\n", - " import tensorflow as tf\n", - "\n", - " print(\"We are using Tensorflow version {}\".format(tf.__version__))\n", - "\n", - " major_version = int(tf.__version__.split(\".\")[0])\n", - " if major_version >= 2:\n", - " from tensorflow.python.util import _pywrap_util_port\n", - " print(\"Intel-optimizations (DNNL) enabled:\",\n", - " _pywrap_util_port.IsMklEnabled())\n", - " else:\n", - " print(\"Intel-optimizations (DNNL) enabled:\")\n", - "\n", - "test_intel_tensorflow()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('keras_cnn_mnist')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Import and process training, validation, and test images/labels\n", - "\n", - "# Set the ratio of validation imgs, can't be 0.0\n", - "VALID_PERCENT = 0.3\n", - "\n", - "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n", - "split_on = int((1 - VALID_PERCENT) * len(X_train))\n", - "\n", - "train_images = X_train[0:split_on,:,:]\n", - "train_labels = to_categorical(y_train)[0:split_on,:]\n", - "\n", - "valid_images = X_train[split_on:,:,:]\n", - "valid_labels = to_categorical(y_train)[split_on:,:]\n", - "\n", - "test_images = X_test\n", - "test_labels = to_categorical(y_test)\n", - "\n", - "def preprocess(images):\n", - " #Normalize\n", - " images = (images / 255) - 0.5\n", - " images = images.reshape(images.shape[0], -1)\n", - "# images = np.expand_dims(images, axis=-1)\n", - " return images\n", - "\n", - "# Preprocess the images.\n", - "train_images = preprocess(train_images)\n", - "valid_images = preprocess(valid_images)\n", - "test_images = preprocess(test_images)\n", - "\n", - "feature_shape = train_images.shape[1:]\n", - "classes = 10\n", - "\n", - "class UnbalancedFederatedDataset(FederatedDataSet):\n", - " def split(self, num_collaborators, shuffle=True, equally=False):\n", - " train_idx = self.split_lognormal(self.y_train, num_collaborators)\n", - " X_train = np.array([self.X_train[idx] for idx in train_idx])\n", - " y_train = np.array([self.y_train[idx] for idx in train_idx])\n", - " \n", - " valid_idx = self.split_lognormal(self.y_valid, num_collaborators)\n", - " X_valid = np.array([self.X_valid[idx] for idx in valid_idx])\n", - " y_valid = np.array([self.y_valid[idx] for idx in valid_idx])\n", - " \n", - " return [\n", - " FederatedDataSet(\n", - " X_train[i],\n", - " y_train[i],\n", - " X_valid[i],\n", - " y_valid[i],\n", - " batch_size=self.batch_size,\n", - " num_classes=self.num_classes\n", - " ) for i in range(num_collaborators)\n", - " ]\n", - " \n", - " def split_lognormal(self, labels, num_collaborators):\n", - " from tqdm import trange\n", - " labels = np.argmax(labels, axis=1)\n", - " idx = [[np.nonzero(labels == (col + j) % self.num_classes)[0][np.arange(5) + (col // 10 * 10 + 5 * j)] \\\n", - " for j in range(2)] for col in range(num_collaborators)]\n", - " idx = [np.hstack(tup) for tup in idx]\n", - " assert all([len(i) == 10 for i in idx]), 'All collaborators should have 10 elements at this stage'\n", - " props = np.random.lognormal(0, 2.0, (10,100,2))\n", - " props = np.array([[[len(np.nonzero(labels==label)[0])-1000]] for label in range(10)])*props/np.sum(props,(1,2), keepdims=True)\n", - " #idx = 1000*np.ones(10, dtype=np.int64)\n", - " for user in trange(1000):\n", - " for j in range(2):\n", - " l = (user+j)%10\n", - " num_samples = int(props[l,user//10,j])\n", - " if np.count_nonzero(labels[np.hstack(idx)] == l) + num_samples < len(np.nonzero(labels==l)[0]):\n", - " idx_to_append = np.nonzero(labels == (user + j) % 10)[0][np.arange(num_samples) + np.count_nonzero(labels[np.hstack(idx)] == l)]\n", - " idx[user] = np.append(idx[user], idx_to_append)\n", - " return idx\n", - "\n", - "fl_data = UnbalancedFederatedDataset(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.utilities.optimizers.keras import FedProxOptimizer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def build_model(input_shape,\n", - " num_classes,\n", - " **kwargs):\n", - " \"\"\"\n", - " Define the model architecture.\n", - "\n", - " Args:\n", - " input_shape (numpy.ndarray): The shape of the data\n", - " num_classes (int): The number of classes of the dataset\n", - "\n", - " Returns:\n", - " tensorflow.python.keras.engine.sequential.Sequential: The model defined in Keras\n", - "\n", - " \"\"\"\n", - " model = Sequential()\n", - " \n", - " model.add(tf.keras.Input(shape=input_shape))\n", - " model.add(Dense(num_classes, activation='softmax'))\n", - "\n", - " model.compile(loss=keras.losses.categorical_crossentropy,\n", - " optimizer=FedProxOptimizer(mu=1),\n", - " metrics=['accuracy'])\n", - "\n", - " return model " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Create a federated model using the build model function and dataset\n", - "fl_model = FederatedModel(build_model, data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=1000)\n", - " \n", - "collaborators = {f'col{col}':collaborator_models[col] for col in range(len(collaborator_models))}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original MNIST dataset\n", - "print(f'Original training data size: {len(train_images)}')\n", - "print(f'Original validation data size: {len(valid_images)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "print(f'Collaborator one\\'s training data size: {len(collaborator_models[0].data_loader.X_train)}')\n", - "print(f'Collaborator one\\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator two's data\n", - "print(f'Collaborator two\\'s training data size: {len(collaborator_models[1].data_loader.X_train)}')\n", - "print(f'Collaborator two\\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Get the current values of the plan. Each of these can be overridden\n", - "import json\n", - "print(json.dumps(fx.get_plan(), indent=4, sort_keys=True))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(collaborators,override_config={'aggregator.settings.rounds_to_train':5, 'collaborator.settings.opt_treatment': 'CONTINUE_GLOBAL'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model and load into keras\n", - "final_fl_model.save_native('final_model')\n", - "model = tf.keras.models.load_model('./final_model')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Test the final model on our test set\n", - "model.evaluate(test_images,test_labels)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "plt.figure(figsize=(9,6), dpi=150)\n", - "plt.title('Keras MNIST unbalanced split')\n", - "plt.plot([0.07627802075538784, 0.07518334008473902, 0.09541350667830556, 0.13141966053564103, 0.15887578643299638], label='FedAvg')\n", - "plt.plot([0.07627802075538784, 0.07518334008473902, 0.09541350667830556, 0.1314459763141349, 0.15887578643299638], linestyle='--', label='FedProx (mu=1e-2)')\n", - "plt.plot([0.07627802075538784, 0.0751056043850258, 0.09555227747093886, 0.131649036151357, 0.15966261748969554], linestyle='--', label='FedProx (mu=1e-1)')\n", - "plt.plot([0.07627802075538784, 0.07517912408802659, 0.09641592293512076, 0.13676991989742965, 0.1684917744528502], linestyle='--', label='FedProx (mu=1e1)')\n", - "\n", - "plt.legend()\n", - "plt.xticks(range(5))\n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_FedProx_PyTorch_MNIST_Tutorial.ipynb b/openfl-tutorials/Federated_FedProx_PyTorch_MNIST_Tutorial.ipynb deleted file mode 100644 index 8f6c23f6c9..0000000000 --- a/openfl-tutorials/Federated_FedProx_PyTorch_MNIST_Tutorial.ipynb +++ /dev/null @@ -1,523 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated FedProx PyTorch MNIST Tutorial\n", - "The only difference between this notebook and Federated_Pytorch_MNIST_Tutorial.ipynb is overriding of the `train_epoch` function in model definition. [See details](#FedProx)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Install dependencies if not already installed\n", - "!pip install torch torchvision" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "\n", - "import torchvision\n", - "import torchvision.transforms as transforms\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel,FederatedDataSet\n", - "import random\n", - "import warnings\n", - "warnings.filterwarnings('ignore')\n", - "def set_seed(seed):\n", - " torch.manual_seed(seed)\n", - " torch.cuda.manual_seed_all(seed)\n", - " torch.backends.cudnn.deterministic = True\n", - " torch.backends.cudnn.benchmark = False\n", - " np.random.seed(seed)\n", - " random.seed(seed)\n", - "set_seed(10)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('torch_cnn_mnist')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def one_hot(labels, classes):\n", - " return np.eye(classes)[labels]\n", - "\n", - "transform = transforms.Compose(\n", - " [transforms.ToTensor(),\n", - " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", - "\n", - "trainset = torchvision.datasets.MNIST(root='./data', train=True,\n", - " download=True, transform=transform)\n", - "\n", - "train_images,train_labels = trainset.train_data, np.array(trainset.train_labels)\n", - "train_images = torch.from_numpy(np.expand_dims(train_images, axis=1)).float()\n", - "train_labels = one_hot(train_labels,10)\n", - "\n", - "validset = torchvision.datasets.MNIST(root='./data', train=False,\n", - " download=True, transform=transform)\n", - "\n", - "valid_images,valid_labels = validset.test_data, np.array(validset.test_labels)\n", - "valid_images = torch.from_numpy(np.expand_dims(valid_images, axis=1)).float()\n", - "valid_labels = one_hot(valid_labels,10)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# FedProx" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.utilities.optimizers.torch import FedProxOptimizer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feature_shape = train_images.shape[1]\n", - "classes = 10\n", - "\n", - "fl_data = FederatedDataSet(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes)\n", - "\n", - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.conv1 = nn.Conv2d(1, 16, 3)\n", - " self.pool = nn.MaxPool2d(2, 2)\n", - " self.conv2 = nn.Conv2d(16, 32, 3)\n", - " self.fc1 = nn.Linear(32 * 5 * 5, 32)\n", - " self.fc2 = nn.Linear(32, 84)\n", - " self.fc3 = nn.Linear(84, 10)\n", - "\n", - " def forward(self, x):\n", - " x = self.pool(F.relu(self.conv1(x)))\n", - " x = self.pool(F.relu(self.conv2(x)))\n", - " x = x.view(x.size(0),-1)\n", - " x = F.relu(self.fc1(x))\n", - " x = F.relu(self.fc2(x))\n", - " x = self.fc3(x)\n", - " return F.log_softmax(x, dim=1)\n", - " \n", - " def train_epoch(self, batch_generator):\n", - " from openfl.federated.task import PyTorchTaskRunner\n", - " self.optimizer.set_old_weights([p for p in self.parameters()])\n", - " return PyTorchTaskRunner.train_epoch(self, batch_generator)\n", - "\n", - " \n", - "optimizer = lambda x: FedProxOptimizer(x, lr=1e-3, mu=0.1)\n", - "\n", - "def cross_entropy(output, target):\n", - " \"\"\"Binary cross-entropy metric\n", - " \"\"\"\n", - " return F.binary_cross_entropy_with_logits(input=output,target=target.float())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "#Create a federated model using the pytorch class, lambda optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy,data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=2)\n", - "collaborators = {'one':collaborator_models[0],'two':collaborator_models[1]}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original MNIST dataset\n", - "print(f'Original training data size: {len(train_images)}')\n", - "print(f'Original validation data size: {len(valid_images)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "print(f'Collaborator one\\'s training data size: {len(collaborator_models[0].data_loader.X_train)}')\n", - "print(f'Collaborator one\\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator two's data\n", - "print(f'Collaborator two\\'s training data size: {len(collaborator_models[1].data_loader.X_train)}')\n", - "print(f'Collaborator two\\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " #Get the current values of the plan. Each of these can be overridden\n", - "import json\n", - "print(json.dumps(fx.get_plan(), indent=4, sort_keys=True))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(\n", - " collaborators,\n", - " {\n", - " 'aggregator.settings.rounds_to_train': 5,\n", - " 'collaborator.settings.opt_treatment': 'CONTINUE_GLOBAL',\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model\n", - "final_fl_model.save_native('final_pytorch_model')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# FedProxAdam" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "code_folding": [] - }, - "outputs": [], - "source": [ - "classes = 10\n", - "collaborator_num = 300\n", - "NUM_USER = collaborator_num\n", - "\n", - "def one_hot(labels, classes):\n", - " return np.eye(classes)[labels]\n", - "\n", - "\n", - "def softmax(x):\n", - " ex = np.exp(x)\n", - " sum_ex = np.sum(np.exp(x))\n", - " return ex/sum_ex\n", - "\n", - "\n", - "def generate_synthetic(alpha, beta, iid):\n", - "\n", - " dimension = 60\n", - " NUM_CLASS = 10\n", - "\n", - " samples_per_user = np.random.lognormal(4, 2, (NUM_USER)).astype(int) + 50\n", - " num_samples = np.sum(samples_per_user)\n", - "\n", - " X_split = [[] for _ in range(NUM_USER)]\n", - " y_split = [[] for _ in range(NUM_USER)]\n", - "\n", - " #### define some eprior ####\n", - " mean_W = np.random.normal(0, alpha, NUM_USER)\n", - " mean_b = mean_W\n", - " B = np.random.normal(0, beta, NUM_USER)\n", - " mean_x = np.zeros((NUM_USER, dimension))\n", - "\n", - " diagonal = np.zeros(dimension)\n", - " for j in range(dimension):\n", - " diagonal[j] = np.power((j+1), -1.2)\n", - " cov_x = np.diag(diagonal)\n", - "\n", - " for i in range(NUM_USER):\n", - " if iid == 1:\n", - " mean_x[i] = np.ones(dimension) * B[i] # all zeros\n", - " else:\n", - " mean_x[i] = np.random.normal(B[i], 1, dimension)\n", - "\n", - " if iid == 1:\n", - " W_global = np.random.normal(0, 1, (dimension, NUM_CLASS))\n", - " b_global = np.random.normal(0, 1, NUM_CLASS)\n", - "\n", - " for i in range(NUM_USER):\n", - "\n", - " W = np.random.normal(mean_W[i], 1, (dimension, NUM_CLASS))\n", - " b = np.random.normal(mean_b[i], 1, NUM_CLASS)\n", - "\n", - " if iid == 1:\n", - " W = W_global\n", - " b = b_global\n", - "\n", - " xx = np.random.multivariate_normal(\n", - " mean_x[i], cov_x, samples_per_user[i])\n", - " yy = np.zeros(samples_per_user[i])\n", - "\n", - " for j in range(samples_per_user[i]):\n", - " tmp = np.dot(xx[j], W) + b\n", - " yy[j] = np.argmax(softmax(tmp))\n", - "\n", - " X_split[i] = xx.tolist()\n", - " y_split[i] = yy.tolist()\n", - "\n", - "# print(\"{}-th users has {} exampls\".format(i, len(y_split[i])))\n", - "\n", - " return X_split, y_split\n", - "\n", - "\n", - "class SyntheticFederatedDataset(FederatedDataSet):\n", - " def __init__(self, batch_size=1, num_classes=None, **kwargs):\n", - " X, y = generate_synthetic(0.0, 0.0, 0)\n", - " X = [np.array([np.array(sample).astype(np.float32)\n", - " for sample in col]) for col in X]\n", - " y = [np.array([np.array(one_hot(int(sample), classes))\n", - " for sample in col]) for col in y]\n", - " self.X_train_all = np.array([col[:int(0.9 * len(col))] for col in X])\n", - " self.X_valid_all = np.array([col[int(0.9 * len(col)):] for col in X])\n", - " self.y_train_all = np.array([col[:int(0.9 * len(col))] for col in y])\n", - " self.y_valid_all = np.array([col[int(0.9 * len(col)):] for col in y])\n", - " super().__init__(self.X_train_all[0], self.y_train_all[0], self.X_valid_all[0],\n", - " self.y_valid_all[0], batch_size, num_classes)\n", - "\n", - " def split(self, num_collaborators, shuffle=True, equally=False):\n", - " return [\n", - " FederatedDataSet(\n", - " self.X_train_all[i],\n", - " self.y_train_all[i],\n", - " self.X_valid_all[i],\n", - " self.y_valid_all[i],\n", - " batch_size=self.batch_size,\n", - " num_classes=self.num_classes\n", - " ) for i in range(num_collaborators)\n", - " ]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.utilities.optimizers.torch import FedProxAdam " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.linear1 = nn.Linear(60, 100)\n", - " self.linear2 = nn.Linear(100, 10)\n", - "\n", - " def forward(self, x):\n", - " x = self.linear1(x)\n", - " x = self.linear2(x)\n", - " return x\n", - "\n", - " def train_epoch(self, batch_generator):\n", - " from openfl.federated.task import PyTorchTaskRunner\n", - " self.optimizer.set_old_weights(\n", - " [p.clone().detach() for p in self.parameters()])\n", - " return PyTorchTaskRunner.train_epoch(self, batch_generator)\n", - "\n", - "\n", - "def optimizer(x): return FedProxAdam(x, lr=1e-3, mu=0.01)\n", - "# optimizer = lambda x: torch.optim.Adam(x, lr=1e-3)\n", - "\n", - "\n", - "def cross_entropy(output, target):\n", - " \"\"\"Binary cross-entropy metric\n", - " \"\"\"\n", - " return F.cross_entropy(output, torch.max(target, 1)[1])\n", - "# return F.binary_cross_entropy_with_logits(input=output,target=target.float())\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fl_data = SyntheticFederatedDataset(batch_size=32, num_classes=classes)\n", - "#Create a federated model using the pytorch class, lambda optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy,data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=collaborator_num,device='cpu')\n", - "collaborators = {f'col{i}':collaborator_models[i] for i in range(collaborator_num)}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "a = np.argmax(collaborators['col3'].data_loader.y_valid, axis =1)\n", - "import matplotlib.pyplot as plt\n", - "plt.hist(a)\n", - "collaborator_models[1].data_loader.y_valid.shape" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(\n", - " collaborators,\n", - " {\n", - " 'aggregator.settings.rounds_to_train': 20,\n", - " 'collaborator.settings.opt_treatment': 'CONTINUE_GLOBAL',\n", - " }\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model\n", - "final_fl_model.save_native('final_pytorch_model')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb b/openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb deleted file mode 100644 index fbdab4b46e..0000000000 --- a/openfl-tutorials/Federated_Keras_MNIST_Tutorial.ipynb +++ /dev/null @@ -1,280 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated Keras MNIST Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "#Install Tensorflow and MNIST dataset if not installed\n", - "!pip install tensorflow==2.13\n", - "\n", - "#Alternatively you could use the intel-tensorflow build\n", - "# !pip install intel-tensorflow==2.13" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import tensorflow as tf\n", - "import tensorflow.keras as keras\n", - "from tensorflow.keras import backend as K\n", - "from tensorflow.keras import Sequential\n", - "from tensorflow.keras.layers import Conv2D, Flatten, Dense\n", - "from tensorflow.keras.utils import to_categorical\n", - "from tensorflow.keras.datasets import mnist\n", - "\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel,FederatedDataSet" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def test_intel_tensorflow():\n", - " \"\"\"\n", - " Check if Intel version of TensorFlow is installed\n", - " \"\"\"\n", - " import tensorflow as tf\n", - "\n", - " print(\"We are using Tensorflow version {}\".format(tf.__version__))\n", - "\n", - " major_version = int(tf.__version__.split(\".\")[0])\n", - " if major_version >= 2:\n", - " from tensorflow.python.util import _pywrap_util_port\n", - " print(\"Intel-optimizations (DNNL) enabled:\",\n", - " _pywrap_util_port.IsMklEnabled())\n", - " else:\n", - " print(\"Intel-optimizations (DNNL) enabled:\")\n", - "\n", - "test_intel_tensorflow()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('keras_cnn_mnist')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Import and process training, validation, and test images/labels\n", - "\n", - "# Set the ratio of validation imgs, can't be 0.0\n", - "VALID_PERCENT = 0.3\n", - "\n", - "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n", - "split_on = int((1 - VALID_PERCENT) * len(X_train))\n", - "\n", - "train_images = X_train[0:split_on,:,:]\n", - "train_labels = to_categorical(y_train)[0:split_on,:]\n", - "\n", - "valid_images = X_train[split_on:,:,:]\n", - "valid_labels = to_categorical(y_train)[split_on:,:]\n", - "\n", - "test_images = X_test\n", - "test_labels = to_categorical(y_test)\n", - "\n", - "def preprocess(images):\n", - " #Normalize\n", - " images = (images / 255) - 0.5\n", - " #Flatten\n", - " images = images.reshape((-1, 784))\n", - " return images\n", - "\n", - "# Preprocess the images.\n", - "train_images = preprocess(train_images)\n", - "valid_images = preprocess(valid_images)\n", - "test_images = preprocess(test_images)\n", - "\n", - "feature_shape = train_images.shape[1]\n", - "classes = 10\n", - "\n", - "fl_data = FederatedDataSet(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes)\n", - "\n", - "def build_model(feature_shape,classes):\n", - " #Defines the MNIST model\n", - " model = Sequential()\n", - " model.add(Dense(64, input_shape=feature_shape, activation='relu'))\n", - " model.add(Dense(64, activation='relu'))\n", - " model.add(Dense(classes, activation='softmax'))\n", - " \n", - " model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'],)\n", - " return model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Create a federated model using the build model function and dataset\n", - "fl_model = FederatedModel(build_model,data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=2)\n", - "collaborators = {'one':collaborator_models[0],'two':collaborator_models[1]}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original MNIST dataset\n", - "print(f'Original training data size: {len(train_images)}')\n", - "print(f'Original validation data size: {len(valid_images)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "print(f'Collaborator one\\'s training data size: {len(collaborator_models[0].data_loader.X_train)}')\n", - "print(f'Collaborator one\\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator two's data\n", - "print(f'Collaborator two\\'s training data size: {len(collaborator_models[1].data_loader.X_train)}')\n", - "print(f'Collaborator two\\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Get the current values of the plan. Each of these can be overridden\n", - "print(fx.get_plan())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(collaborators,override_config={'aggregator.settings.rounds_to_train':5})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model and load into keras\n", - "final_fl_model.save_native('final_model')\n", - "model = tf.keras.models.load_model('./final_model')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Test the final model on our test set\n", - "model.evaluate(test_images,test_labels)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_PyTorch_TinyImageNet.ipynb b/openfl-tutorials/Federated_PyTorch_TinyImageNet.ipynb deleted file mode 100644 index b526806bc3..0000000000 --- a/openfl-tutorials/Federated_PyTorch_TinyImageNet.ipynb +++ /dev/null @@ -1,378 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated PyTorch TinyImageNet Tutorial" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This notebook is an example of Transfer Learning \n", - "\n", - "Custom DataLoader is used with OpenFL Python API" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Install dependencies if not already installed\n", - "!pip install torch torchvision\n", - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import glob\n", - "from torch.utils.data import Dataset, DataLoader\n", - "from PIL import Image\n", - "\n", - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "\n", - "import torchvision\n", - "from torchvision import transforms as T\n", - "\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel, FederatedDataSet" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('torch_cnn_mnist')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Download the data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget --no-clobber http://cs231n.stanford.edu/tiny-imagenet-200.zip\n", - "!unzip -n tiny-imagenet-200.zip\n", - "TINY_IMAGENET_ROOT = './tiny-imagenet-200/'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Describe the dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class TinyImageNet(Dataset):\n", - " \"\"\"\n", - " Contains 200 classes for training. Each class has 500 images. \n", - " Parameters\n", - " ----------\n", - " root: string\n", - " Root directory including `train` and `val` subdirectories.\n", - " split: string\n", - " Indicating which split to return as a data set.\n", - " Valid option: [`train`, `val`]\n", - " transform: torchvision.transforms\n", - " A (series) of valid transformation(s).\n", - " \"\"\"\n", - " def __init__(self, root, split='train', transform=None, target_transform=None):\n", - " NUM_IMAGES_PER_CLASS = 500\n", - " self.root = os.path.expanduser(root)\n", - " self.transform = transform\n", - " self.target_transform = target_transform\n", - " self.split_dir = os.path.join(self.root, split)\n", - " self.image_paths = sorted(glob.iglob(os.path.join(self.split_dir, '**', '*.JPEG'), recursive=True))\n", - " \n", - " self.labels = {} # fname - label number mapping\n", - "\n", - " # build class label - number mapping\n", - " with open(os.path.join(self.root, 'wnids.txt'), 'r') as fp:\n", - " self.label_texts = sorted([text.strip() for text in fp.readlines()])\n", - " self.label_text_to_number = {text: i for i, text in enumerate(self.label_texts)}\n", - "\n", - " if split == 'train':\n", - " for label_text, i in self.label_text_to_number.items():\n", - " for cnt in range(NUM_IMAGES_PER_CLASS):\n", - " self.labels[f'{label_text}_{cnt}.JPEG'] = i\n", - " elif split == 'val':\n", - " with open(os.path.join(self.split_dir, 'val_annotations.txt'), 'r') as fp:\n", - " for line in fp.readlines():\n", - " terms = line.split('\\t')\n", - " file_name, label_text = terms[0], terms[1]\n", - " self.labels[file_name] = self.label_text_to_number[label_text]\n", - " \n", - " \n", - " def __len__(self):\n", - " return len(self.image_paths)\n", - "\n", - " def __getitem__(self, index):\n", - " file_path = self.image_paths[index]\n", - " label = self.labels[os.path.basename(file_path)]\n", - " label = self.target_transform(label) if self.target_transform else label\n", - " return self.read_image(file_path), label\n", - "\n", - " def read_image(self, path):\n", - " img = Image.open(path)\n", - " return self.transform(img) if self.transform else img\n", - "\n", - "def one_hot(labels, classes):\n", - " return np.eye(classes)[labels]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "normalize = T.Normalize(mean=[0.485, 0.456, 0.406],\n", - " std=[0.229, 0.224, 0.225])\n", - "\n", - "augmentation = T.RandomApply([\n", - " T.RandomHorizontalFlip(),\n", - " T.RandomRotation(10),\n", - " T.RandomResizedCrop(64)], p=.8)\n", - "\n", - "training_transform = T.Compose([\n", - " T.Lambda(lambda x: x.convert(\"RGB\")),\n", - " augmentation,\n", - " T.ToTensor(),\n", - " normalize])\n", - "\n", - "valid_transform = T.Compose([\n", - " T.Lambda(lambda x: x.convert(\"RGB\")),\n", - " T.ToTensor(),\n", - " normalize])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Implement Federated dataset\n", - "We have to implement `split` method" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.utilities.data_splitters import EqualNumPyDataSplitter\n", - "from torch.utils.data import Subset\n", - "\n", - "\n", - "train_set = TinyImageNet(TINY_IMAGENET_ROOT, 'train', transform=training_transform)\n", - "valid_set = TinyImageNet(TINY_IMAGENET_ROOT, 'val', transform=valid_transform, \\\n", - " target_transform=lambda target: one_hot(target, 200))\n", - "\n", - "class TinyImageNetFederatedDataset(DataLoader):\n", - " def __init__(self, train_set, valid_set, batch_size):\n", - " self.data_splitter = EqualNumPyDataSplitter()\n", - " self.train_set = train_set\n", - " self.valid_set = valid_set\n", - " self.batch_size = batch_size\n", - " \n", - " def split(self, num_collaborators):\n", - " train_split = self.data_splitter.split([label for _, label in self.train_set], num_collaborators)\n", - " valid_split = self.data_splitter.split([label for _, label in self.valid_set], num_collaborators)\n", - " return [\n", - " TinyImageNetFederatedDataset(\n", - " Subset(self.train_set, train_split[i]),\n", - " Subset(self.valid_set, valid_split[i]),\n", - " self.batch_size\n", - " )\n", - " for i in range(num_collaborators)\n", - " ]\n", - " \n", - " def get_feature_shape(self):\n", - " return self.train_set[0][0].shape\n", - " \n", - " def get_train_loader(self, num_batches=None):\n", - " return DataLoader(self.train_set, batch_size=self.batch_size)\n", - " \n", - " def get_valid_loader(self):\n", - " return DataLoader(self.valid_set)\n", - " \n", - " def get_train_data_size(self):\n", - " return len(self.train_set)\n", - " \n", - " def get_valid_data_size(self):\n", - " return len(self.valid_set)\n", - " \n", - "fl_data = TinyImageNetFederatedDataset(train_set, valid_set, batch_size=32)\n", - "\n", - "num_classes = 200" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Define model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.model = torchvision.models.mobilenet_v2(pretrained=True)\n", - " self.model.requires_grad_(False)\n", - " self.model.classifier[1] = torch.nn.Linear(in_features=1280, \\\n", - " out_features=num_classes, bias=True)\n", - "\n", - " def forward(self, x):\n", - " x = self.model.forward(x)\n", - " return x\n", - "\n", - " \n", - "optimizer = lambda x: optim.Adam(x, lr=1e-4)\n", - "\n", - "def cross_entropy(output, target):\n", - " \"\"\"Binary cross-entropy metric\n", - " \"\"\"\n", - " return F.cross_entropy(input=output,target=target)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Create a federated model using the pytorch class, lambda optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy, \\\n", - " data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=10)\n", - "collaborators = {'one':collaborator_models[0],'two':collaborator_models[1]}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original TinyImageNet dataset\n", - "print(f'Original training data size: {len(fl_data.train_set)}')\n", - "print(f'Original validation data size: {len(fl_data.valid_set)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "for i, model in enumerate(collaborator_models):\n", - " print(f'Collaborator {i}\\'s training data size: {len(model.data_loader.train_set)}')\n", - " print(f'Collaborator {i}\\'s validation data size: {len(model.data_loader.valid_set)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(collaborators,{'aggregator.settings.rounds_to_train':10})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model\n", - "final_fl_model.save_native('final_model.pth')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_PyTorch_UNET_Tutorial.ipynb b/openfl-tutorials/Federated_PyTorch_UNET_Tutorial.ipynb deleted file mode 100644 index 7ee6c2e692..0000000000 --- a/openfl-tutorials/Federated_PyTorch_UNET_Tutorial.ipynb +++ /dev/null @@ -1,545 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated PyTorch UNET Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Install dependencies if not already installed\n", - "!pip install torch" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First of all we need to set up our OpenFL workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import openfl.native as fx\n", - "\n", - "# Setup default workspace, logging, etc. Install additional requirements\n", - "fx.init('torch_unet_kvasir')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import installed modules\n", - "import PIL\n", - "import json\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "import numpy as np\n", - "from skimage import io\n", - "from torchvision import transforms as tsf\n", - "import matplotlib.pyplot as plt\n", - "from torch.utils.data import Dataset, DataLoader\n", - "\n", - "from os import listdir\n", - "\n", - "from openfl.federated import FederatedModel, FederatedDataSet\n", - "from openfl.utilities import TensorKey\n", - "from openfl.utilities import validate_file_hash" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Download Kvasir dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!wget 'https://datasets.simula.no/downloads/hyper-kvasir/hyper-kvasir-segmented-images.zip' -O kvasir.zip\n", - "ZIP_SHA384 = ('66cd659d0e8afd8c83408174'\n", - " '1ade2b75dada8d4648b816f2533c8748b1658efa3d49e205415d4116faade2c5810e241e')\n", - "validate_file_hash('./kvasir.zip', ZIP_SHA384)\n", - "!unzip -n kvasir.zip -d ./data" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "DATA_PATH = './data/segmented-images/'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def read_data(image_path, mask_path):\n", - " \"\"\"\n", - " Read image and mask from disk.\n", - " \"\"\"\n", - " img = io.imread(image_path)\n", - " assert(img.shape[2] == 3)\n", - " mask = io.imread(mask_path)\n", - " return (img, mask[:, :, 0].astype(np.uint8))\n", - "\n", - "\n", - "class KvasirDataset(Dataset):\n", - " \"\"\"\n", - " Kvasir dataset contains 1000 images for all collaborators.\n", - " Args:\n", - " data_path: path to dataset on disk\n", - " collaborator_count: total number of collaborators\n", - " collaborator_num: number of current collaborator\n", - " is_validation: validation option\n", - " \"\"\"\n", - "\n", - " def __init__(self, data_path, collaborator_count, collaborator_num, is_validation):\n", - " self.images_path = './data/segmented-images/images/'\n", - " self.masks_path = './data/segmented-images/masks/'\n", - " self.images_names = [\n", - " img_name\n", - " for img_name in sorted(listdir(self.images_path))\n", - " if len(img_name) > 3 and img_name[-3:] == 'jpg'\n", - " ]\n", - "\n", - " self.images_names = self.images_names[collaborator_num:: collaborator_count]\n", - " self.is_validation = is_validation\n", - " assert(len(self.images_names) > 8)\n", - " validation_size = len(self.images_names) // 8\n", - " if is_validation:\n", - " self.images_names = self.images_names[-validation_size:]\n", - " else:\n", - " self.images_names = self.images_names[: -validation_size]\n", - "\n", - " self.img_trans = tsf.Compose([\n", - " tsf.ToPILImage(),\n", - " tsf.Resize((332, 332)),\n", - " tsf.ToTensor(),\n", - " tsf.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])])\n", - " self.mask_trans = tsf.Compose([\n", - " tsf.ToPILImage(),\n", - " tsf.Resize((332, 332), interpolation=PIL.Image.NEAREST),\n", - " tsf.ToTensor()])\n", - "\n", - " def __getitem__(self, index):\n", - " name = self.images_names[index]\n", - " img, mask = read_data(self.images_path + name, self.masks_path + name)\n", - " img = self.img_trans(img).numpy()\n", - " mask = self.mask_trans(mask).numpy()\n", - " return img, mask\n", - "\n", - " def __len__(self):\n", - " return len(self.images_names)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we redefine `FederatedDataSet` methods, if we don't want to use default batch generator from `FederatedDataSet`. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class KvasirFederatedDataset(FederatedDataSet):\n", - " def __init__(self, collaborator_count=1, collaborator_num=0, batch_size=1, **kwargs):\n", - " \"\"\"Instantiate the data object\n", - " Args:\n", - " collaborator_count: total number of collaborators\n", - " collaborator_num: number of current collaborator\n", - " batch_size: the batch size of the data loader\n", - " **kwargs: additional arguments, passed to super init\n", - " \"\"\"\n", - " super().__init__([], [], [], [], batch_size, num_classes=2, **kwargs)\n", - "\n", - " self.collaborator_num = int(collaborator_num)\n", - "\n", - " self.batch_size = batch_size\n", - "\n", - " self.training_set = KvasirDataset(\n", - " DATA_PATH, collaborator_count, collaborator_num, is_validation=False\n", - " )\n", - " self.valid_set = KvasirDataset(\n", - " DATA_PATH, collaborator_count, collaborator_num, is_validation=True\n", - " )\n", - "\n", - " self.train_loader = self.get_train_loader()\n", - " self.val_loader = self.get_valid_loader()\n", - "\n", - " def get_valid_loader(self, num_batches=None):\n", - " return DataLoader(self.valid_set, num_workers=8, batch_size=self.batch_size)\n", - "\n", - " def get_train_loader(self, num_batches=None):\n", - " return DataLoader(\n", - " self.training_set, num_workers=8, batch_size=self.batch_size, shuffle=True\n", - " )\n", - "\n", - " def get_train_data_size(self):\n", - " return len(self.training_set)\n", - "\n", - " def get_valid_data_size(self):\n", - " return len(self.valid_set)\n", - "\n", - " def get_feature_shape(self):\n", - " return self.valid_set[0][0].shape\n", - "\n", - " def split(self, collaborator_count, shuffle=True, equally=True):\n", - " return [\n", - " KvasirFederatedDataset(collaborator_count,\n", - " collaborator_num, self.batch_size)\n", - " for collaborator_num in range(collaborator_count)\n", - " ]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Our Unet model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def soft_dice_loss(output, target):\n", - " num = target.size(0)\n", - " m1 = output.view(num, -1)\n", - " m2 = target.view(num, -1)\n", - " intersection = m1 * m2\n", - " score = 2.0 * (intersection.sum(1) + 1) / (m1.sum(1) + m2.sum(1) + 1)\n", - " score = 1 - score.sum() / num\n", - " return score\n", - "\n", - "\n", - "def soft_dice_coef(output, target):\n", - " num = target.size(0)\n", - " m1 = output.view(num, -1)\n", - " m2 = target.view(num, -1)\n", - " intersection = m1 * m2\n", - " score = 2.0 * (intersection.sum(1) + 1) / (m1.sum(1) + m2.sum(1) + 1)\n", - " return score.sum()\n", - "\n", - "\n", - "class DoubleConv(nn.Module):\n", - " def __init__(self, in_ch, out_ch):\n", - " super(DoubleConv, self).__init__()\n", - " self.in_ch = in_ch\n", - " self.out_ch = out_ch\n", - " self.conv = nn.Sequential(\n", - " nn.Conv2d(in_ch, out_ch, 3, padding=1),\n", - " nn.BatchNorm2d(out_ch),\n", - " nn.ReLU(inplace=True),\n", - " nn.Conv2d(out_ch, out_ch, 3, padding=1),\n", - " nn.BatchNorm2d(out_ch),\n", - " nn.ReLU(inplace=True),\n", - " )\n", - "\n", - " def forward(self, x):\n", - " x = self.conv(x)\n", - " return x\n", - "\n", - "\n", - "class Down(nn.Module):\n", - " def __init__(self, in_ch, out_ch):\n", - " super(Down, self).__init__()\n", - " self.mpconv = nn.Sequential(\n", - " nn.MaxPool2d(2),\n", - " DoubleConv(in_ch, out_ch)\n", - " )\n", - "\n", - " def forward(self, x):\n", - " x = self.mpconv(x)\n", - " return x\n", - "\n", - "\n", - "class Up(nn.Module):\n", - " def __init__(self, in_ch, out_ch, bilinear=False):\n", - " super(Up, self).__init__()\n", - " self.in_ch = in_ch\n", - " self.out_ch = out_ch\n", - " if bilinear:\n", - " self.Up = nn.Upsample(\n", - " scale_factor=2,\n", - " mode=\"bilinear\",\n", - " align_corners=True\n", - " )\n", - " else:\n", - " self.Up = nn.ConvTranspose2d(in_ch, in_ch // 2, 2, stride=2)\n", - " self.conv = DoubleConv(in_ch, out_ch)\n", - "\n", - " def forward(self, x1, x2):\n", - " x1 = self.Up(x1)\n", - " diffY = x2.size()[2] - x1.size()[2]\n", - " diffX = x2.size()[3] - x1.size()[3]\n", - "\n", - " x1 = F.pad(x1, (diffX // 2, diffX - diffX //\n", - " 2, diffY // 2, diffY - diffY // 2))\n", - "\n", - " x = torch.cat([x2, x1], dim=1)\n", - " x = self.conv(x)\n", - " return x\n", - "\n", - "\n", - "class UNet(nn.Module):\n", - " def __init__(self, n_channels=3, n_classes=1):\n", - " super().__init__()\n", - " self.inc = DoubleConv(n_channels, 64)\n", - " self.down1 = Down(64, 128)\n", - " self.down2 = Down(128, 256)\n", - " self.down3 = Down(256, 512)\n", - " self.down4 = Down(512, 1024)\n", - " self.up1 = Up(1024, 512)\n", - " self.up2 = Up(512, 256)\n", - " self.up3 = Up(256, 128)\n", - " self.up4 = Up(128, 64)\n", - " self.outc = nn.Conv2d(64, n_classes, 1)\n", - "\n", - " def forward(self, x):\n", - " x1 = self.inc(x)\n", - " x2 = self.down1(x1)\n", - " x3 = self.down2(x2)\n", - " x4 = self.down3(x3)\n", - " x5 = self.down4(x4)\n", - " x = self.up1(x5, x4)\n", - " x = self.up2(x, x3)\n", - " x = self.up3(x, x2)\n", - " x = self.up4(x, x1)\n", - " x = self.outc(x)\n", - " x = torch.sigmoid(x)\n", - " return x\n", - "\n", - " def validate(\n", - " self, col_name, round_num, input_tensor_dict, use_tqdm=False, **kwargs\n", - " ):\n", - " \"\"\" Validate. Redifine function from PyTorchTaskRunner, to use our validation\"\"\"\n", - " self.rebuild_model(round_num, input_tensor_dict, validation=True)\n", - " self.eval()\n", - " self.to(self.device)\n", - " val_score = 0\n", - " total_samples = 0\n", - "\n", - " loader = self.data_loader.get_valid_loader()\n", - " if use_tqdm:\n", - " loader = tqdm.tqdm(loader, desc=\"validate\")\n", - "\n", - " with torch.no_grad():\n", - " for data, target in loader:\n", - " samples = target.shape[0]\n", - " total_samples += samples\n", - " data, target = (\n", - " torch.tensor(data).to(self.device),\n", - " torch.tensor(target).to(self.device),\n", - " )\n", - " output = self(data)\n", - " # get the index of the max log-probability\n", - " val = soft_dice_coef(output, target)\n", - " val_score += val.sum().cpu().numpy()\n", - "\n", - " origin = col_name\n", - " suffix = \"validate\"\n", - " if kwargs[\"apply\"] == \"local\":\n", - " suffix += \"_local\"\n", - " else:\n", - " suffix += \"_agg\"\n", - " tags = (\"metric\", suffix)\n", - " output_tensor_dict = {\n", - " TensorKey(\"dice_coef\", origin, round_num, True, tags): np.array(\n", - " val_score / total_samples\n", - " )\n", - " }\n", - " return output_tensor_dict, {}\n", - "\n", - "\n", - "def optimizer(x): return optim.Adam(x, lr=1e-3)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create `KvasirFederatedDataset`, federated datasets for collaborators will be created in `split()` method of this object" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "fl_data = KvasirFederatedDataset(batch_size=6)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with OpenFL. It provides built-in federated training function which will be used while training. Using its `setup` function, collaborator models and datasets can be automatically obtained for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Create a federated model using the pytorch class, optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=UNet, optimizer=optimizer,\n", - " loss_fn=soft_dice_loss, data_loader=fl_data)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=2)\n", - "collaborators = {'one': collaborator_models[0], 'two': collaborator_models[1]}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current FL plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Get the current values of the FL plan. Each of these can be overridden\n", - "print(fx.get_plan())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom FL plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(\n", - " collaborators, override_config={'aggregator.settings.rounds_to_train': 30})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Save final model\n", - "final_fl_model.save_native('final_pytorch_model')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's visually evaluate the results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator = collaborator_models[0]\n", - "loader = collaborator.runner.data_loader.get_valid_loader()\n", - "model = final_fl_model.model\n", - "model.eval()\n", - "device = final_fl_model.runner.device\n", - "model.to(device)\n", - "with torch.no_grad():\n", - " for batch, _ in zip(loader, range(5)):\n", - " preds = model(batch[0].to(device))\n", - " for image, pred, target in zip(batch[0], preds, batch[1]):\n", - " plt.figure(figsize=(10, 10))\n", - " plt.subplot(131)\n", - " plt.imshow(image.permute(1, 2, 0).data.cpu().numpy() * 0.5 + 0.5)\n", - " plt.title(\"img\")\n", - " plt.subplot(132)\n", - " plt.imshow(pred[0].data.cpu().numpy())\n", - " plt.title(\"pred\")\n", - " plt.subplot(133)\n", - " plt.imshow(target[0].data.cpu().numpy())\n", - " plt.title(\"targ\")\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.10 64-bit", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.8.10" - }, - "vscode": { - "interpreter": { - "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_Pytorch_MNIST_Tutorial.ipynb b/openfl-tutorials/Federated_Pytorch_MNIST_Tutorial.ipynb deleted file mode 100644 index 10e83949df..0000000000 --- a/openfl-tutorials/Federated_Pytorch_MNIST_Tutorial.ipynb +++ /dev/null @@ -1,267 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated PyTorch MNIST Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Install dependencies if not already installed\n", - "!pip install torch torchvision" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "\n", - "import torchvision\n", - "import torchvision.transforms as transforms\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel,FederatedDataSet\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('torch_cnn_mnist', log_level='METRIC', log_file='./spam_metric.log')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "transform = transforms.Compose(\n", - " [transforms.ToTensor(),\n", - " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", - "\n", - "trainset = torchvision.datasets.MNIST(root='./data', train=True,\n", - " download=True, transform=transform)\n", - "\n", - "train_images,train_labels = trainset.data, np.array(trainset.targets)\n", - "train_images = torch.from_numpy(np.expand_dims(train_images, axis=1)).float()\n", - "\n", - "validset = torchvision.datasets.MNIST(root='./data', train=False,\n", - " download=True, transform=transform)\n", - "\n", - "valid_images,valid_labels = validset.data, np.array(validset.targets)\n", - "valid_images = torch.from_numpy(np.expand_dims(valid_images, axis=1)).float()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feature_shape = train_images.shape[1]\n", - "classes = 10\n", - "\n", - "fl_data = FederatedDataSet(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes)\n", - "\n", - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.conv1 = nn.Conv2d(1, 16, 3)\n", - " self.pool = nn.MaxPool2d(2, 2)\n", - " self.conv2 = nn.Conv2d(16, 32, 3)\n", - " self.fc1 = nn.Linear(32 * 5 * 5, 32)\n", - " self.fc2 = nn.Linear(32, 84)\n", - " self.fc3 = nn.Linear(84, 10)\n", - "\n", - " def forward(self, x):\n", - " x = self.pool(F.relu(self.conv1(x)))\n", - " x = self.pool(F.relu(self.conv2(x)))\n", - " x = x.view(x.size(0),-1)\n", - " x = F.relu(self.fc1(x))\n", - " x = F.relu(self.fc2(x))\n", - " x = self.fc3(x)\n", - " return F.log_softmax(x, dim=1)\n", - " \n", - "optimizer = lambda x: optim.Adam(x, lr=1e-4)\n", - "\n", - "def cross_entropy(output, target):\n", - " \"\"\"Binary cross-entropy metric\n", - " \"\"\"\n", - " return F.cross_entropy(input=output,target=target)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we can define metric logging function. It should has the following signature described below. You can use it to write metrics to tensorboard or some another specific logging." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from torch.utils.tensorboard import SummaryWriter\n", - "\n", - "writer = SummaryWriter('./logs/cnn_mnist', flush_secs=5)\n", - "\n", - "\n", - "def write_metric(node_name, task_name, metric_name, metric, round_number):\n", - " writer.add_scalar(\"{}/{}/{}\".format(node_name, task_name, metric_name),\n", - " metric, round_number)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "#Create a federated model using the pytorch class, lambda optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy,data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=2)\n", - "collaborators = {'one':collaborator_models[0],'two':collaborator_models[1]}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original MNIST dataset\n", - "print(f'Original training data size: {len(train_images)}')\n", - "print(f'Original validation data size: {len(valid_images)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "print(f'Collaborator one\\'s training data size: {len(collaborator_models[0].data_loader.X_train)}')\n", - "print(f'Collaborator one\\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator two's data\n", - "print(f'Collaborator two\\'s training data size: {len(collaborator_models[1].data_loader.X_train)}')\n", - "print(f'Collaborator two\\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " #Get the current values of the plan. Each of these can be overridden\n", - "print(fx.get_plan())" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to run our experiment. If we want to pass in custom plan settings, we can easily do that with the `override_config` parameter" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Run experiment, return trained FederatedModel\n", - "\n", - "final_fl_model = fx.run_experiment(collaborators, override_config={\n", - " 'aggregator.settings.rounds_to_train': 5,\n", - " 'aggregator.settings.log_metric_callback': write_metric,\n", - "})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model\n", - "final_fl_model.save_native('final_pytorch_model')" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.1" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-tutorials/Federated_Pytorch_MNIST_custom_aggregation_Tutorial.ipynb b/openfl-tutorials/Federated_Pytorch_MNIST_custom_aggregation_Tutorial.ipynb deleted file mode 100644 index 337ceb3259..0000000000 --- a/openfl-tutorials/Federated_Pytorch_MNIST_custom_aggregation_Tutorial.ipynb +++ /dev/null @@ -1,708 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Federated PyTorch MNIST Tutorial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Install dependencies if not already installed\n", - "!pip install torch torchvision" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "\n", - "import torchvision\n", - "import torchvision.transforms as transforms\n", - "import openfl.native as fx\n", - "from openfl.federated import FederatedModel,FederatedDataSet\n", - "\n", - "torch.manual_seed(0)\n", - "np.random.seed(0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "After importing the required packages, the next step is setting up our openfl workspace. To do this, simply run the `fx.init()` command as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Setup default workspace, logging, etc.\n", - "fx.init('torch_cnn_mnist')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we are ready to define our dataset and model to perform federated learning on. The dataset should be composed of a numpy arrayWe start with a simple fully connected model that is trained on the MNIST dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def one_hot(labels, classes):\n", - " return np.eye(classes)[labels]\n", - "\n", - "transform = transforms.Compose(\n", - " [transforms.ToTensor(),\n", - " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", - "\n", - "trainset = torchvision.datasets.MNIST(root='./data', train=True,\n", - " download=True, transform=transform)\n", - "\n", - "train_images,train_labels = trainset.train_data, np.array(trainset.train_labels)\n", - "train_images = torch.from_numpy(np.expand_dims(train_images, axis=1)).float()\n", - "train_labels = one_hot(train_labels,10)\n", - "\n", - "validset = torchvision.datasets.MNIST(root='./data', train=False,\n", - " download=True, transform=transform)\n", - "\n", - "valid_images,valid_labels = validset.test_data, np.array(validset.test_labels)\n", - "valid_images = torch.from_numpy(np.expand_dims(valid_images, axis=1)).float()\n", - "valid_labels = one_hot(valid_labels,10)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "feature_shape = train_images.shape[1]\n", - "classes = 10\n", - "\n", - "fl_data = FederatedDataSet(train_images,train_labels,valid_images,valid_labels,batch_size=32,num_classes=classes)\n", - "\n", - "class Net(nn.Module):\n", - " def __init__(self):\n", - " super(Net, self).__init__()\n", - " self.conv1 = nn.Conv2d(1, 16, 3)\n", - " self.pool = nn.MaxPool2d(2, 2)\n", - " self.conv2 = nn.Conv2d(16, 32, 3)\n", - " self.fc1 = nn.Linear(32 * 5 * 5, 32)\n", - " self.fc2 = nn.Linear(32, 84)\n", - " self.fc3 = nn.Linear(84, 10)\n", - "\n", - " def forward(self, x):\n", - " x = self.pool(F.relu(self.conv1(x)))\n", - " x = self.pool(F.relu(self.conv2(x)))\n", - " x = x.view(x.size(0),-1)\n", - " x = F.relu(self.fc1(x))\n", - " x = F.relu(self.fc2(x))\n", - " x = self.fc3(x)\n", - " return x\n", - " \n", - "optimizer = lambda x: optim.Adam(x, lr=1e-4)\n", - "\n", - "def cross_entropy(output, target):\n", - " \"\"\"Binary cross-entropy metric\n", - " \"\"\"\n", - " return F.cross_entropy(input=output,target=torch.argmax(target, dim=1))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "#Create a federated model using the pytorch class, lambda optimizer function, and loss function\n", - "fl_model = FederatedModel(build_model=Net,optimizer=optimizer,loss_fn=cross_entropy,data_loader=fl_data)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The `FederatedModel` object is a wrapper around your Keras, Tensorflow or PyTorch model that makes it compatible with openfl. It provides built in federated training and validation functions that we will see used below. Using it's `setup` function, collaborator models and datasets can be automatically defined for the experiment. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "collaborator_models = fl_model.setup(num_collaborators=10)\n", - "collaborators = {str(i): collaborator_models[i] for i in range(10)}#, 'three':collaborator_models[2]}" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Original MNIST dataset\n", - "print(f'Original training data size: {len(train_images)}')\n", - "print(f'Original validation data size: {len(valid_images)}\\n')\n", - "\n", - "#Collaborator one's data\n", - "print(f'Collaborator one\\'s training data size: {len(collaborator_models[0].data_loader.X_train)}')\n", - "print(f'Collaborator one\\'s validation data size: {len(collaborator_models[0].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator two's data\n", - "print(f'Collaborator two\\'s training data size: {len(collaborator_models[1].data_loader.X_train)}')\n", - "print(f'Collaborator two\\'s validation data size: {len(collaborator_models[1].data_loader.X_valid)}\\n')\n", - "\n", - "#Collaborator three's data\n", - "#print(f'Collaborator three\\'s training data size: {len(collaborator_models[2].data_loader.X_train)}')\n", - "#print(f'Collaborator three\\'s validation data size: {len(collaborator_models[2].data_loader.X_valid)}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can see the current plan values by running the `fx.get_plan()` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " #Get the current values of the plan. Each of these can be overridden\n", - "print(fx.get_plan())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.interface.aggregation_functions import AggregationFunction\n", - "import numpy as np\n", - "\n", - "class ExponentialSmoothingAveraging(AggregationFunction):\n", - " \"\"\"\n", - " Averaging via exponential smoothing.\n", - " \n", - " In order to use this mechanism properly you should specify `aggregator.settings.db_store_rounds` \n", - " in `override_config` keyword argument of `run_experiment` function. \n", - " It should be equal to the number of rounds you want to include in smoothing window.\n", - " \n", - " Args:\n", - " alpha(float): Smoothing term.\n", - " \"\"\"\n", - " def __init__(self, alpha=0.9):\n", - " self.alpha = alpha\n", - " \n", - " def call(self,\n", - " local_tensors,\n", - " db_iterator,\n", - " tensor_name,\n", - " fl_round,\n", - " tags):\n", - " \"\"\"Aggregate tensors.\n", - "\n", - " Args:\n", - " local_tensors(list[openfl.utilities.LocalTensor]): List of local tensors to aggregate.\n", - " db_iterator: iterator over history of all tensors. Columns:\n", - " - 'tensor_name': name of the tensor.\n", - " Examples for `torch.nn.Module`s: 'conv1.weight', 'fc2.bias'.\n", - " - 'round': 0-based number of round corresponding to this tensor.\n", - " - 'tags': tuple of tensor tags. Tags that can appear:\n", - " - 'model' indicates that the tensor is a model parameter.\n", - " - 'trained' indicates that tensor is a part of a training result.\n", - " These tensors are passed to the aggregator node after local learning.\n", - " - 'aggregated' indicates that tensor is a result of aggregation.\n", - " These tensors are sent to collaborators for the next round.\n", - " - 'delta' indicates that value is a difference between rounds\n", - " for a specific tensor.\n", - " also one of the tags is a collaborator name\n", - " if it corresponds to a result of a local task.\n", - "\n", - " - 'nparray': value of the tensor.\n", - " tensor_name: name of the tensor\n", - " fl_round: round number\n", - " tags: tuple of tags for this tensor\n", - " Returns:\n", - " np.ndarray: aggregated tensor\n", - " \"\"\"\n", - " tensors, weights = zip(*[(x.tensor, x.weight) for x in local_tensors])\n", - " tensors, weights = np.array(tensors), np.array(weights)\n", - " average = np.average(tensors, weights=weights, axis=0)\n", - " previous_tensor_values = []\n", - " for record in db_iterator:\n", - " if (\n", - " record['tensor_name'] == tensor_name\n", - " and 'aggregated' in record['tags']\n", - " and 'delta' not in record['tags']\n", - " ):\n", - " previous_tensor_values.append(record['nparray'])\n", - " for i, x in enumerate(previous_tensor_values):\n", - " previous_tensor_values[i] = x * self.alpha * (1 - self.alpha) ** i\n", - " smoothing_term = np.sum(previous_tensor_values, axis=0)\n", - " return self.alpha * average + (1 - self.alpha) * smoothing_term" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.interface.aggregation_functions import AggregationFunction\n", - "import numpy as np\n", - "\n", - "class ClippedAveraging(AggregationFunction):\n", - " def __init__(self, ratio):\n", - " \"\"\"Average clipped tensors.\n", - " \n", - " Args:\n", - " ratio(float): Ratio to multiply with a tensor for clipping\n", - " \"\"\"\n", - " self.ratio = ratio\n", - " \n", - " def call(self,\n", - " local_tensors,\n", - " db_iterator,\n", - " tensor_name,\n", - " fl_round,\n", - " *__):\n", - " \"\"\"Aggregate tensors.\n", - "\n", - " Args:\n", - " local_tensors(list[openfl.utilities.LocalTensor]): List of local tensors to aggregate.\n", - " db_iterator: iterator over history of all tensors. Columns:\n", - " - 'tensor_name': name of the tensor.\n", - " Examples for `torch.nn.Module`s: 'conv1.weight', 'fc2.bias'.\n", - " - 'round': 0-based number of round corresponding to this tensor.\n", - " - 'tags': tuple of tensor tags. Tags that can appear:\n", - " - 'model' indicates that the tensor is a model parameter.\n", - " - 'trained' indicates that tensor is a part of a training result.\n", - " These tensors are passed to the aggregator node after local learning.\n", - " - 'aggregated' indicates that tensor is a result of aggregation.\n", - " These tensors are sent to collaborators for the next round.\n", - " - 'delta' indicates that value is a difference between rounds\n", - " for a specific tensor.\n", - " also one of the tags is a collaborator name\n", - " if it corresponds to a result of a local task.\n", - "\n", - " - 'nparray': value of the tensor.\n", - " tensor_name: name of the tensor\n", - " fl_round: round number\n", - " tags: tuple of tags for this tensor\n", - " Returns:\n", - " np.ndarray: aggregated tensor\n", - " \"\"\"\n", - " clipped_tensors = []\n", - " previous_tensor_value = None\n", - " for record in db_iterator:\n", - " if (\n", - " record['round'] == (fl_round - 1)\n", - " and record['tensor_name'] == tensor_name\n", - " and record['tags'] == ('trained',)\n", - " ):\n", - " previous_tensor_value = record['nparray']\n", - " weights = []\n", - " for local_tensor in local_tensors:\n", - " prev_tensor = previous_tensor_value if previous_tensor_value is not None else local_tensor.tensor\n", - " delta = local_tensor.tensor - prev_tensor\n", - " new_tensor = prev_tensor + delta * self.ratio\n", - " clipped_tensors.append(new_tensor)\n", - " weights.append(local_tensor.weight)\n", - "\n", - " return np.average(clipped_tensors, weights=weights, axis=0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.interface.aggregation_functions import AggregationFunction\n", - "\n", - "class ConditionalThresholdAveraging(AggregationFunction):\n", - " def __init__(self, threshold_fn, metric_name='acc', tags=['metric', 'validate_local']):\n", - " \"\"\"Average tensors by metric value on previous round.\n", - " If no tensors match threshold condition, a simple weighted averaging will be performed.\n", - " \n", - " Args:\n", - " threshold_fn(callable): function to define a threshold for each round.\n", - " Has single argument `round_number`. \n", - " Returns threshold value above which collaborators are allowed to participate in aggregation.\n", - " metric_name(str): name of the metric to trace. Can be either 'acc' or 'loss'.\n", - " tags(Tuple[str]): tags of the metric tensor.\n", - " \"\"\"\n", - " self.metric_name = metric_name\n", - " self.threshold_fn = threshold_fn\n", - " self.tags = tags\n", - " self.logged_round = -1\n", - " \n", - " def call(self,\n", - " local_tensors,\n", - " db_iterator,\n", - " tensor_name,\n", - " fl_round,\n", - " *__):\n", - " \"\"\"Aggregate tensors.\n", - "\n", - " Args:\n", - " local_tensors(list[openfl.utilities.LocalTensor]): List of local tensors to aggregate.\n", - " db_iterator: iterator over history of all tensors. Columns:\n", - " - 'tensor_name': name of the tensor.\n", - " Examples for `torch.nn.Module`s: 'conv1.weight', 'fc2.bias'.\n", - " - 'round': 0-based number of round corresponding to this tensor.\n", - " - 'tags': tuple of tensor tags. Tags that can appear:\n", - " - 'model' indicates that the tensor is a model parameter.\n", - " - 'trained' indicates that tensor is a part of a training result.\n", - " These tensors are passed to the aggregator node after local learning.\n", - " - 'aggregated' indicates that tensor is a result of aggregation.\n", - " These tensors are sent to collaborators for the next round.\n", - " - 'delta' indicates that value is a difference between rounds\n", - " for a specific tensor.\n", - " also one of the tags is a collaborator name\n", - " if it corresponds to a result of a local task.\n", - "\n", - " - 'nparray': value of the tensor.\n", - " tensor_name: name of the tensor\n", - " fl_round: round number\n", - " tags: tuple of tags for this tensor\n", - " Returns:\n", - " np.ndarray: aggregated tensor\n", - " \"\"\"\n", - " selected_tensors = []\n", - " selected_weights = []\n", - " for record in db_iterator:\n", - " for local_tensor in local_tensors:\n", - " tags = set(self.tags + [local_tensor.col_name])\n", - " if (\n", - " tags <= set(record['tags']) \n", - " and record['round'] == fl_round\n", - " and record['tensor_name'] == self.metric_name\n", - " and record['nparray'] >= self.threshold_fn(fl_round)\n", - " ):\n", - " selected_tensors.append(local_tensor.tensor)\n", - " selected_weights.append(local_tensor.weight)\n", - " if not selected_tensors:\n", - " if self.logged_round < fl_round:\n", - " fx.logger.warning('No collaborators match threshold condition. Performing simple averaging...')\n", - " selected_tensors = [local_tensor.tensor for local_tensor in local_tensors]\n", - " selected_weights = [local_tensor.weight for local_tensor in local_tensors]\n", - " if self.logged_round < fl_round:\n", - " self.logged_round += 1\n", - " return np.average(selected_tensors, weights=selected_weights, axis=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Privileged Aggregation Functions\n", - "Most of the time the `AggregationFunction` interface is sufficient to implement custom methods, but in certain scenarios users may want to store additional information inside the TensorDB Dataframe beyond the aggregated tensor. The `openfl.interface.aggregation_functions.experimental.PrivilegedAggregationFunction` interface is provided for this use, and gives the user direct access to aggregator's TensorDB dataframe (notice the `tensor_db` param in the call function replaces the `db_iterator` from the standard AggregationFunction interface). As the name suggests, this interface is called privileged because with great power comes great responsibility, and modifying the TensorDB dataframe directly can lead to unexpected behavior and experiment failures if entries are arbitrarily deleted.\n", - "\n", - "Note that in-place methods (`.loc`) on the tensor_db dataframe are required for write operations. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from openfl.interface.aggregation_functions.experimental import PrivilegedAggregationFunction\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "class PrioritizeLeastImproved(PrivilegedAggregationFunction):\n", - " \"\"\"\n", - " Give collaborator with the least improvement in validation accuracy more influence over future weights\n", - " \n", - " \"\"\"\n", - " \n", - " def call(self,\n", - " local_tensors,\n", - " tensor_db,\n", - " tensor_name,\n", - " fl_round,\n", - " tags):\n", - " \"\"\"Aggregate tensors.\n", - "\n", - " Args:\n", - " local_tensors(list[openfl.utilities.LocalTensor]): List of local tensors to aggregate.\n", - " tensor_db: Aggregator's TensorDB [writable]. Columns:\n", - " - 'tensor_name': name of the tensor.\n", - " Examples for `torch.nn.Module`s: 'conv1.weight', 'fc2.bias'.\n", - " - 'round': 0-based number of round corresponding to this tensor.\n", - " - 'tags': tuple of tensor tags. Tags that can appear:\n", - " - 'model' indicates that the tensor is a model parameter.\n", - " - 'trained' indicates that tensor is a part of a training result.\n", - " These tensors are passed to the aggregator node after local learning.\n", - " - 'aggregated' indicates that tensor is a result of aggregation.\n", - " These tensors are sent to collaborators for the next round.\n", - " - 'delta' indicates that value is a difference between rounds\n", - " for a specific tensor.\n", - " also one of the tags is a collaborator name\n", - " if it corresponds to a result of a local task.\n", - "\n", - " - 'nparray': value of the tensor.\n", - " tensor_name: name of the tensor\n", - " fl_round: round number\n", - " tags: tuple of tags for this tensor\n", - " Returns:\n", - " np.ndarray: aggregated tensor\n", - " \"\"\"\n", - " from openfl.utilities import change_tags\n", - "\n", - " tensors, weights, collaborators = zip(*[(x.tensor, x.weight, x.col_name) for idx,x in enumerate(local_tensors)])\n", - " tensors, weights, collaborators = np.array(tensors), np.array(weights), collaborators\n", - "\n", - " if fl_round > 0:\n", - " metric_tags = ('metric','validate_agg')\n", - " collaborator_accuracy = {}\n", - " previous_col_accuracy = {}\n", - " change_in_accuracy = {}\n", - " for col in collaborators:\n", - " col_metric_tag = change_tags(metric_tags,add_field=col)\n", - " collaborator_accuracy[col] = float(tensor_db[(tensor_db['tensor_name'] == 'acc') &\n", - " (tensor_db['round'] == fl_round) &\n", - " (tensor_db['tags'] == col_metric_tag)]['nparray'])\n", - " previous_col_accuracy[col] = float(tensor_db[(tensor_db['tensor_name'] == 'acc') &\n", - " (tensor_db['round'] == fl_round - 1) &\n", - " (tensor_db['tags'] == col_metric_tag)]['nparray'])\n", - " change_in_accuracy[col] = collaborator_accuracy[col] - previous_col_accuracy[col]\n", - " \n", - " \n", - " least_improved_collaborator = min(change_in_accuracy,key=change_in_accuracy.get)\n", - " \n", - " # Dont add least improved collaborator more than once\n", - " if len(tensor_db[(tensor_db['tags'] == ('least_improved',)) &\n", - " (tensor_db['round'] == fl_round)]) == 0:\n", - " tensor_db.loc[tensor_db.shape[0]] = \\\n", - " ['_','_',fl_round,True,('least_improved',),np.array(least_improved_collaborator)]\n", - " least_improved_weight_factor = 0.1 * len(tensor_db[(tensor_db['tags'] == ('least_improved',)) &\n", - " (tensor_db['nparray'] == np.array(least_improved_collaborator))])\n", - " weights[collaborators.index(least_improved_collaborator)] += least_improved_weight_factor\n", - " weights = weights / np.sum(weights)\n", - " \n", - " return np.average(tensors, weights=weights, axis=0)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To make the process of writing, reading from, and searching through dataframes easier, we add three methods to the tensor_db dataframe. `store`, `retrieve`, and `search`. Power users can still use all of the built-in pandas dataframe methods, but because some prior knowledge is needed to effectively deal with dataframe column types, iterating through them, and how to store them in a consistent way that won't break other OpenFL functionality, these three methods provide a conventient way to let researchers focus on algorithms instead internal framework machinery. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class FedAvgM_Selection(PrivilegedAggregationFunction):\n", - " \"\"\"\n", - " Adapted from FeTS Challenge 2021\n", - " Federated Brain Tumor Segmentation:Multi-Institutional Privacy-Preserving Collaborative Learning\n", - " Ece Isik-Polat, Gorkem Polat,Altan Kocyigit1, and Alptekin Temizel1\n", - " \n", - " \"\"\"\n", - " \n", - " def call(\n", - " self,\n", - " local_tensors,\n", - " tensor_db,\n", - " tensor_name,\n", - " fl_round,\n", - " tags):\n", - " \n", - " \"\"\"Aggregate tensors.\n", - "\n", - " Args:\n", - " local_tensors(list[openfl.utilities.LocalTensor]): List of local tensors to aggregate.\n", - " tensor_db: Aggregator's TensorDB [writable]. Columns:\n", - " - 'tensor_name': name of the tensor.\n", - " Examples for `torch.nn.Module`s: 'conv1.weight', 'fc2.bias'.\n", - " - 'round': 0-based number of round corresponding to this tensor.\n", - " - 'tags': tuple of tensor tags. Tags that can appear:\n", - " - 'model' indicates that the tensor is a model parameter.\n", - " - 'trained' indicates that tensor is a part of a training result.\n", - " These tensors are passed to the aggregator node after local learning.\n", - " - 'aggregated' indicates that tensor is a result of aggregation.\n", - " These tensors are sent to collaborators for the next round.\n", - " - 'delta' indicates that value is a difference between rounds\n", - " for a specific tensor.\n", - " also one of the tags is a collaborator name\n", - " if it corresponds to a result of a local task.\n", - "\n", - " - 'nparray': value of the tensor.\n", - " tensor_name: name of the tensor\n", - " fl_round: round number\n", - " tags: tuple of tags for this tensor\n", - " Returns:\n", - " np.ndarray: aggregated tensor\n", - " \"\"\"\n", - " #momentum\n", - " tensor_db.store(tensor_name='momentum',nparray=0.9,overwrite=False)\n", - " #aggregator_lr\n", - " tensor_db.store(tensor_name='aggregator_lr',nparray=1.0,overwrite=False)\n", - "\n", - " if fl_round == 0:\n", - " # Just apply FedAvg\n", - "\n", - " tensor_values = [t.tensor for t in local_tensors]\n", - " weight_values = [t.weight for t in local_tensors] \n", - " new_tensor_weight = np.average(tensor_values, weights=weight_values, axis=0) \n", - "\n", - " #if not (tensor_name in weight_speeds):\n", - " if tensor_name not in tensor_db.search(tags=('weight_speeds',))['tensor_name']: \n", - " #weight_speeds[tensor_name] = np.zeros_like(local_tensors[0].tensor) # weight_speeds[tensor_name] = np.zeros(local_tensors[0].tensor.shape)\n", - " tensor_db.store(\n", - " tensor_name=tensor_name, \n", - " tags=('weight_speeds',), \n", - " nparray=np.zeros_like(local_tensors[0].tensor),\n", - " )\n", - " return new_tensor_weight \n", - " else:\n", - " if tensor_name.endswith(\"weight\") or tensor_name.endswith(\"bias\"):\n", - " # Calculate aggregator's last value\n", - " previous_tensor_value = None\n", - " for _, record in tensor_db.iterrows():\n", - " if (record['round'] == fl_round \n", - " and record[\"tensor_name\"] == tensor_name\n", - " and record[\"tags\"] == (\"aggregated\",)): \n", - " previous_tensor_value = record['nparray']\n", - " break\n", - "\n", - " if previous_tensor_value is None:\n", - " logger.warning(\"Error in fedAvgM: previous_tensor_value is None\")\n", - " logger.warning(\"Tensor: \" + tensor_name)\n", - "\n", - " # Just apply FedAvg \n", - " tensor_values = [t.tensor for t in local_tensors]\n", - " weight_values = [t.weight for t in local_tensors] \n", - " new_tensor_weight = np.average(tensor_values, weights=weight_values, axis=0) \n", - " \n", - " if tensor_name not in tensor_db.search(tags=('weight_speeds',))['tensor_name']: \n", - " tensor_db.store(\n", - " tensor_name=tensor_name, \n", - " tags=('weight_speeds',), \n", - " nparray=np.zeros_like(local_tensors[0].tensor),\n", - " )\n", - "\n", - " return new_tensor_weight\n", - " else:\n", - " # compute the average delta for that layer\n", - " deltas = [previous_tensor_value - t.tensor for t in local_tensors]\n", - " weight_values = [t.weight for t in local_tensors]\n", - " average_deltas = np.average(deltas, weights=weight_values, axis=0) \n", - "\n", - " # V_(t+1) = momentum*V_t + Average_Delta_t\n", - " tensor_weight_speed = tensor_db.retrieve(\n", - " tensor_name=tensor_name,\n", - " tags=('weight_speeds',)\n", - " )\n", - " \n", - " momentum = float(tensor_db.retrieve(tensor_name='momentum'))\n", - " aggregator_lr = float(tensor_db.retrieve(tensor_name='aggregator_lr'))\n", - " \n", - " new_tensor_weight_speed = momentum * tensor_weight_speed + average_deltas # fix delete (1-momentum)\n", - " \n", - " tensor_db.store(\n", - " tensor_name=tensor_name, \n", - " tags=('weight_speeds',), \n", - " nparray=new_tensor_weight_speed\n", - " )\n", - " # W_(t+1) = W_t-lr*V_(t+1)\n", - " new_tensor_weight = previous_tensor_value - aggregator_lr*new_tensor_weight_speed\n", - "\n", - " return new_tensor_weight\n", - " else:\n", - " # Just apply FedAvg \n", - " tensor_values = [t.tensor for t in local_tensors]\n", - " weight_values = [t.weight for t in local_tensors] \n", - " new_tensor_weight = np.average(tensor_values, weights=weight_values, axis=0)\n", - "\n", - " return new_tensor_weight" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Run experiment, return trained FederatedModel\n", - "final_fl_model = fx.run_experiment(collaborators,\n", - " {\n", - " 'aggregator.settings.rounds_to_train':5,\n", - " 'aggregator.settings.db_store_rounds':5,\n", - " 'tasks.train.aggregation_type': ClippedAveraging(ratio=0.9)\n", - " })" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#Save final model\n", - "final_fl_model.save_native('final_pytorch_model')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "py3.7", - "language": "python", - "name": "py3.7" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.9" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py b/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py index e4e24bf760..3c7746d831 100644 --- a/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py +++ b/openfl-workspace/torch_llm_horovod/src/InHorovodrun.py @@ -9,7 +9,7 @@ import horovod.torch as hvd -import openfl.native as fx +from openfl.interface.cli import setup_logging SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.append(os.path.dirname(SCRIPT_DIR)) @@ -50,7 +50,7 @@ def get_args(): def main(): logger = getLogger(__name__) - fx.setup_logging(level="INFO", log_file=None) + setup_logging() try: logger.info("starting horovod") hvd.init() diff --git a/openfl/native/__init__.py b/openfl/native/__init__.py deleted file mode 100644 index 230ded708d..0000000000 --- a/openfl/native/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -"""openfl.native package.""" - -from openfl.native.native import * # NOQA diff --git a/openfl/native/fastestimator.py b/openfl/native/fastestimator.py deleted file mode 100644 index b794d30387..0000000000 --- a/openfl/native/fastestimator.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -"""FederatedFastEstimator module.""" -import os -from logging import getLogger -from pathlib import Path -from sys import path - -import fastestimator as fe -from fastestimator.trace.io.best_model_saver import BestModelSaver - -import openfl.native as fx -from openfl.federated import Plan -from openfl.federated.data import FastEstimatorDataLoader -from openfl.federated.task import FastEstimatorTaskRunner -from openfl.protocols import utils -from openfl.utilities.split import split_tensor_dict_for_holdouts - - -class FederatedFastEstimator: - """A wrapper for fastestimator.estimator that allows running in federated - mode. - - Attributes: - estimator: The FastEstimator to be used. - logger: A logger to record events. - rounds: The number of rounds to train. - """ - - def __init__(self, estimator, override_config: dict = None, **kwargs): - """Initializes a new instance of the FederatedFastEstimator class. - - Args: - estimator: The FastEstimator to be used. - override_config (dict, optional): A dictionary to override the - default configuration. Defaults to None. - **kwargs: Additional keyword arguments. - """ - self.estimator = estimator - self.logger = getLogger(__name__) - fx.init(**kwargs) - if override_config: - fx.update_plan(override_config) - - def fit(self): - """Runs the estimator in federated mode.""" - - file = Path(__file__).resolve() - # interface root, containing command modules - root = file.parent.resolve() - work = Path.cwd().resolve() - - path.append(str(root)) - path.insert(0, str(work)) - - # TODO: Fix this implementation. The full plan parsing is reused here, - # but the model and data will be overwritten based on - # user specifications - plan_config = Path(fx.WORKSPACE_PREFIX) / "plan" / "plan.yaml" - cols_config = Path(fx.WORKSPACE_PREFIX) / "plan" / "cols.yaml" - data_config = Path(fx.WORKSPACE_PREFIX) / "plan" / "data.yaml" - - plan = Plan.parse( - plan_config_path=plan_config, - cols_config_path=cols_config, - data_config_path=data_config, - ) - - self.rounds = plan.config["aggregator"]["settings"]["rounds_to_train"] - data_loader = FastEstimatorDataLoader(self.estimator.pipeline) - runner = FastEstimatorTaskRunner(self.estimator, data_loader=data_loader) - # Overwrite plan values - tensor_pipe = plan.get_tensor_pipe() - # Initialize model weights - init_state_path = plan.config["aggregator"]["settings"]["init_state_path"] - tensor_dict, holdout_params = split_tensor_dict_for_holdouts( - self.logger, runner.get_tensor_dict(False) - ) - - model_snap = utils.construct_model_proto( - tensor_dict=tensor_dict, round_number=0, tensor_pipe=tensor_pipe - ) - - self.logger.info(f"Creating Initial Weights File" f" 🠆 {init_state_path}") - - utils.dump_proto(model_proto=model_snap, fpath=init_state_path) - - self.logger.info("Starting Experiment...") - - aggregator = plan.get_aggregator() - - model_states = dict.fromkeys(plan.authorized_cols, None) - runners = {} - save_dir = {} - data_path = 1 - for col in plan.authorized_cols: - data = self.estimator.pipeline.data - train_data, eval_data, test_data = split_data( - data["train"], - data["eval"], - data["test"], - data_path, - len(plan.authorized_cols), - ) - pipeline_kwargs = {} - for k, v in self.estimator.pipeline.__dict__.items(): - if k in [ - "batch_size", - "ops", - "num_process", - "drop_last", - "pad_value", - "collate_fn", - ]: - pipeline_kwargs[k] = v - pipeline_kwargs.update( - { - "train_data": train_data, - "eval_data": eval_data, - "test_data": test_data, - } - ) - pipeline = fe.Pipeline(**pipeline_kwargs) - - data_loader = FastEstimatorDataLoader(pipeline) - self.estimator.system.pipeline = pipeline - - runners[col] = FastEstimatorTaskRunner( - estimator=self.estimator, data_loader=data_loader - ) - runners[col].set_optimizer_treatment("CONTINUE_LOCAL") - - for trace in runners[col].estimator.system.traces: - if isinstance(trace, BestModelSaver): - save_dir_path = f"{trace.save_dir}/{col}" - os.makedirs(save_dir_path, exist_ok=True) - save_dir[col] = save_dir_path - - data_path += 1 - - # Create the collaborators - collaborators = { - collaborator: fx.create_collaborator( - plan, collaborator, runners[collaborator], aggregator - ) - for collaborator in plan.authorized_cols - } - - model = None - for round_num in range(self.rounds): - for col in plan.authorized_cols: - collaborator = collaborators[col] - - if round_num != 0: - # For FastEstimator Jupyter notebook, models must be - # saved in different directories (i.e. path must be - # reset here) - - runners[col].estimator.system.load_state(f"save/{col}_state") - runners[col].rebuild_model(round_num, model_states[col]) - - # Reset the save directory if BestModelSaver is present - # in traces - for trace in runners[col].estimator.system.traces: - if isinstance(trace, BestModelSaver): - trace.save_dir = save_dir[col] - - collaborator.run_simulation() - - model_states[col] = runners[col].get_tensor_dict(with_opt_vars=True) - model = runners[col].model - runners[col].estimator.system.save_state(f"save/{col}_state") - - # TODO This will return the model from the last collaborator, - # NOT the final aggregated model (though they should be similar). - # There should be a method added to the aggregator that will load - # the best model from disk and return it - return model - - -def split_data(train, eva, test, rank, collaborator_count): - """Split data into N parts, where N is the collaborator count. - - Args: - train : The training data. - eva : The evaluation data. - test : The testing data. - rank (int): The rank of the current collaborator. - collaborator_count (int): The total number of collaborators. - - Returns: - tuple: The training, evaluation, and testing data for the current - collaborator. - """ - if collaborator_count == 1: - return train, eva, test - - fraction = [1.0 / float(collaborator_count)] - fraction *= collaborator_count - 1 - - # Expand the split list into individual parameters - train_split = train.split(*fraction) - eva_split = eva.split(*fraction) - test_split = test.split(*fraction) - - train = [train] - eva = [eva] - test = [test] - - if type(train_split) is not list: - train.append(train_split) - eva.append(eva_split) - test.append(test_split) - else: - # Combine all partitions into a single list - train = [train] + train_split - eva = [eva] + eva_split - test = [test] + test_split - - # Extract the right shard - train = train[rank - 1] - eva = eva[rank - 1] - test = test[rank - 1] - - return train, eva, test diff --git a/openfl/native/native.py b/openfl/native/native.py deleted file mode 100644 index 88b8c5f426..0000000000 --- a/openfl/native/native.py +++ /dev/null @@ -1,370 +0,0 @@ -# Copyright 2020-2024 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - - -"""OpenFL Native functions module. - -This file defines openfl entrypoints to be used directly through python (not -CLI) -""" -import importlib -import json -import logging -import os -from copy import copy -from logging import basicConfig, getLogger -from pathlib import Path -from sys import path - -import flatten_json -from rich.console import Console -from rich.logging import RichHandler - -import openfl.interface.aggregator as aggregator -import openfl.interface.collaborator as collaborator -import openfl.interface.workspace as workspace -from openfl.federated import Plan -from openfl.protocols import utils -from openfl.utilities import add_log_level -from openfl.utilities.split import split_tensor_dict_for_holdouts - -logger = getLogger(__name__) - -WORKSPACE_PREFIX = os.path.join(os.path.expanduser("~"), ".local", "workspace") - - -def setup_plan(log_level="CRITICAL"): - """ - Dump the plan with all defaults + overrides set. - - Args: - log_level (str, optional): The log level Whether to save the plan to - disk. - Defaults to 'CRITICAL'. - - Returns: - plan: Plan object. - """ - plan_config = "plan/plan.yaml" - cols_config = "plan/cols.yaml" - data_config = "plan/data.yaml" - - current_level = logging.root.level - getLogger().setLevel(log_level) - plan = Plan.parse( - plan_config_path=Path(plan_config), - cols_config_path=Path(cols_config), - data_config_path=Path(data_config), - resolve=False, - ) - getLogger().setLevel(current_level) - - return plan - - -def flatten(config, return_complete=False): - """ - Flatten nested config. - - Args: - config (dict): The configuration dictionary to flatten. - return_complete (bool, optional): Whether to return the complete - flattened config. Defaults to False. - - Returns: - flattened_config (dict): The flattened configuration dictionary. - """ - flattened_config = flatten_json.flatten(config, ".") - if not return_complete: - keys_to_remove = [k for k, v in flattened_config.items() if ("defaults" in k or v is None)] - else: - keys_to_remove = [k for k, v in flattened_config.items() if v is None] - for k in keys_to_remove: - del flattened_config[k] - - return flattened_config - - -def update_plan(override_config, plan=None, resolve=True): - """Updates the plan with the provided override and saves it to disk. - - For a list of available override options, call `fx.get_plan()` - - Args: - override_config (dict): A dictionary of values to override in the plan. - plan (Plan, optional): The plan to update. If None, a new plan is set - up. Defaults to None. - resolve (bool, optional): Whether to resolve the plan. Defaults to - True. - - Returns: - plan (object): The updated plan. - """ - if plan is None: - plan = setup_plan() - flat_plan_config = flatten(plan.config, return_complete=True) - - org_list_keys_with_count = {} - for k in flat_plan_config: - k_split = k.rsplit(".", 1) - if k_split[1].isnumeric(): - if k_split[0] in org_list_keys_with_count: - org_list_keys_with_count[k_split[0]] += 1 - else: - org_list_keys_with_count[k_split[0]] = 1 - - for key, val in override_config.items(): - if key in org_list_keys_with_count: - # remove old list corresponding to this key entirely - for idx in range(org_list_keys_with_count[key]): - del flat_plan_config[f"{key}.{idx}"] - logger.info("Updating %s to %s... ", key, val) - elif key in flat_plan_config: - logger.info("Updating %s to %s... ", key, val) - else: - # TODO: We probably need to validate the new key somehow - logger.info( - "Did not find %s in config. Make sure it should exist. Creating...", - key, - ) - if type(val) is list: - for idx, v in enumerate(val): - flat_plan_config[f"{key}.{idx}"] = v - else: - flat_plan_config[key] = val - - plan.config = unflatten(flat_plan_config, ".") - if resolve: - plan.resolve() - return plan - - -def unflatten(config, separator="."): - """Unfolds `config` settings that have `separator` in their names. - - Args: - config (dict): The flattened configuration dictionary to unfold. - separator (str, optional): The separator used in the flattened config. - Defaults to '.'. - - Returns: - config (dict): The unfolded configuration dictionary. - """ - config = flatten_json.unflatten_list(config, separator) - return config - - -def setup_logging(level="INFO", log_file=None): - """Initializes logging settings. - - Args: - level (str, optional): The log level. Defaults to 'INFO'. - log_file (str, optional): The name of the file to log to. - If None, logs are not saved to a file. Defaults to None. - """ - # Setup logging - - if importlib.util.find_spec("tensorflow") is not None: - import tensorflow as tf # pylint: disable=import-outside-toplevel - - tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) - metric = 25 - add_log_level("METRIC", metric) - - if isinstance(level, str): - level = level.upper() - - handlers = [] - if log_file: - fh = logging.FileHandler(log_file) - formatter = logging.Formatter( - "%(asctime)s %(levelname)s %(message)s %(filename)s:%(lineno)d" - ) - fh.setFormatter(formatter) - handlers.append(fh) - - console = Console(width=160) - handlers.append(RichHandler(console=console)) - basicConfig(level=level, format="%(message)s", datefmt="[%X]", handlers=handlers) - - -def init( - workspace_template: str = "default", - log_level: str = "INFO", - log_file: str = None, - agg_fqdn: str = None, - col_names=None, -): - """ - Initialize the openfl package. - - It performs the following tasks: - - 1. Creates a workspace in ~/.local/workspace (Equivalent to `fx - workspace create --prefix ~/.local/workspace --template - $workspace_template) - 2. Setup certificate authority (equivalent to `fx workspace certify`) - 3. Setup aggregator PKI (equivalent to `fx aggregator - generate-cert-request` followed by `fx aggregator certify`) - 4. Setup list of collaborators (col_names) and their PKI. (Equivalent - to running `fx collaborator generate-cert-request` followed by `fx - collaborator certify` for each of the collaborators in col_names) - 5. Setup logging - - Args: - workspace_template (str): The template that should be used as the - basis for the experiment. Defaults to 'default'. - Other options include are any of the template names - [keras_cnn_mnist, tf_2dunet, tf_cnn_histology, - mtorch_cnn_histology, torch_cnn_mnist]. - log_level (str): Log level for logging. METRIC level is available. - Defaults to 'INFO'. - log_file (str): Name of the file in which the log will be duplicated. - If None, logs are not saved to a file. Defaults to None. - agg_fqdn (str): The local node's fully qualified domain name (if it - can't be resolved automatically). Defaults to None. - col_names (list[str]): The names of the collaborators that will be - created. These collaborators will be set up to participate in the - experiment, but are not required to. Defaults to None. - - Returns: - None - """ - if col_names is None: - col_names = ["one", "two"] - workspace.create(WORKSPACE_PREFIX, workspace_template) - os.chdir(WORKSPACE_PREFIX) - workspace.certify() - aggregator.generate_cert_request(agg_fqdn) - aggregator.certify(agg_fqdn, silent=True) - data_path = 1 - for col_name in col_names: - collaborator.create(col_name, str(data_path), silent=True) - collaborator.generate_cert_request(col_name, silent=True, skip_package=True) - collaborator.certify(col_name, silent=True) - data_path += 1 - - setup_logging(level=log_level, log_file=log_file) - - -def get_collaborator(plan, name, model, aggregator): - """Create the collaborator. - - Using the same plan object to create multiple collaborators leads to - identical collaborator objects. This function can be removed once - collaborator generation is fixed in openfl/federated/plan/plan.py - - Args: - plan (Plan): The plan to use to create the collaborator. - name (str): The name of the collaborator. - model (Model): The model to use for the collaborator. - aggregator (Aggregator): The aggregator to use for the collaborator. - - Returns: - Collaborator: The created collaborator. - """ - plan = copy(plan) - - return plan.get_collaborator(name, task_runner=model, client=aggregator) - - -def run_experiment(collaborator_dict: dict, override_config: dict = None): - """Core function that executes the FL Plan. - - Args: - collaborator_dict (dict): A dictionary mapping collaborator names to - their federated models. - Example: {collaborator_name(str): FederatedModel} - This dictionary defines which collaborators will participate in - the experiment, as well as a reference to that collaborator's - federated model. - override_config (dict, optional): A dictionary of values to override - in the plan. Defaults to None. - Example: dict {flplan.key : flplan.value} - Override any of the plan parameters at runtime using this - dictionary. To get a list of the available options, execute - `fx.get_plan()` - - Returns: - model: Final Federated model. The model resulting from the federated - learning experiment - """ - - if override_config is None: - override_config = {} - - file = Path(__file__).resolve() - root = file.parent.resolve() # interface root, containing command modules - work = Path.cwd().resolve() - - path.append(str(root)) - path.insert(0, str(work)) - - # Update the plan if necessary - plan = update_plan(override_config) - # Overwrite plan values - plan.authorized_cols = list(collaborator_dict) - tensor_pipe = plan.get_tensor_pipe() - - # This must be set to the final index of the list (this is the last - # tensorflow session to get created) - plan.runner_ = list(collaborator_dict.values())[-1] - model = plan.runner_ - - # Initialize model weights - init_state_path = plan.config["aggregator"]["settings"]["init_state_path"] - rounds_to_train = plan.config["aggregator"]["settings"]["rounds_to_train"] - tensor_dict, holdout_params = split_tensor_dict_for_holdouts( - logger, model.get_tensor_dict(False) - ) - - model_snap = utils.construct_model_proto( - tensor_dict=tensor_dict, round_number=0, tensor_pipe=tensor_pipe - ) - - logger.info("Creating Initial Weights File 🠆 %s", init_state_path) - - utils.dump_proto(model_proto=model_snap, fpath=init_state_path) - - logger.info("Starting Experiment...") - - aggregator = plan.get_aggregator() - - # get the collaborators - collaborators = { - collaborator: get_collaborator( - plan, collaborator, collaborator_dict[collaborator], aggregator - ) - for collaborator in plan.authorized_cols - } - - for _ in range(rounds_to_train): - for col in plan.authorized_cols: - collaborator = collaborators[col] - collaborator.run_simulation() - - # Set the weights for the final model - model.rebuild_model(rounds_to_train - 1, aggregator.last_tensor_dict, validation=True) - return model - - -def get_plan(fl_plan=None, indent=4, sort_keys=True): - """Returns a string representation of the current Plan. - - Args: - fl_plan (Plan): The plan to get a string representation of. If None, a - new plan is set up. Defaults to None. - indent (int): The number of spaces to use for indentation in the - string representation. Defaults to 4. - sort_keys (bool): Whether to sort the keys in the string - representation. Defaults to True. - - Returns: - str: A string representation of the plan. - """ - if fl_plan is None: - plan = setup_plan() - else: - plan = fl_plan - flat_plan_config = flatten(plan.config) - return json.dumps(flat_plan_config, indent=indent, sort_keys=sort_keys) diff --git a/setup.py b/setup.py index 947285c108..7a97c46674 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,6 @@ def run(self): 'openfl.federated.task', 'openfl.interface', 'openfl.interface.interactive_api', - 'openfl.native', 'openfl.pipelines', 'openfl.plugins', 'openfl.plugins.frameworks_adapters', diff --git a/tests/github/pki_wrong_cn.py b/tests/github/pki_wrong_cn.py deleted file mode 100644 index 7ce669d707..0000000000 --- a/tests/github/pki_wrong_cn.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -import grpc -import subprocess -import os -import time -from multiprocessing import Process -import sys -import importlib - -import openfl -import openfl.native as fx -from openfl.utilities.utils import getfqdn_env - - -def prepare_workspace(): - subprocess.check_call(['fx', 'workspace', 'certify']) - subprocess.check_call(['fx', 'plan', 'initialize']) - - subprocess.check_call([ - 'fx', 'aggregator', 'generate-cert-request' - ]) - subprocess.check_call([ - 'fx', 'aggregator', 'certify', - '-s' - ]) - for col in ['one', 'two']: - subprocess.check_call([ - 'fx', 'collaborator', 'create', - '-n', col, - '-d', '1', - '-s' - ]) - subprocess.check_call([ - 'fx', 'collaborator', 'generate-cert-request', - '-n', col, - '-s', '-x' - ]) - subprocess.check_call([ - 'fx', 'collaborator', 'certify', - '-n', col, - '-s' - ]) - - sys.path.append(os.getcwd()) - - -def start_invalid_collaborator(): - ''' - We choose the gRPC client of another collaborator - to check if aggregator accepts certificate - that does not correspond to the collaborator's name. - ''' - importlib.reload(openfl.federated.task) # fetch TF-based task runner - importlib.reload(openfl.federated.data) # fetch TF-based data loader - importlib.reload(openfl.federated) # allow imports from parent module - col_name = 'one' - plan = fx.setup_plan() - plan.resolve() - client = plan.get_client('two', plan.aggregator_uuid, plan.federation_uuid) - collaborator = plan.get_collaborator(col_name, client=client) - collaborator.run() - - -def start_aggregator(): - agg = Process(target=subprocess.check_call, args=[['fx', 'aggregator', 'start']]) - agg.start() - time.sleep(3) # wait for initialization - return agg - - -if __name__ == '__main__': - origin_dir = os.getcwd() - prefix = 'fed_workspace' - subprocess.check_call([ - 'fx', 'workspace', 'create', - '--prefix', prefix, - '--template', 'keras_cnn_mnist' - ]) - os.chdir(prefix) - fqdn = getfqdn_env() - prepare_workspace() - agg = start_aggregator() - try: - start_invalid_collaborator() - agg.join() - except grpc.RpcError as e: - if e.code() == grpc.StatusCode.UNAUTHENTICATED: - pass - else: - raise - else: - print('Aggregator accepted invalid collaborator certificate.') - sys.exit(1) - finally: - agg.kill() diff --git a/tests/github/python_native_tf.py b/tests/github/python_native_tf.py deleted file mode 100644 index c5e44f75af..0000000000 --- a/tests/github/python_native_tf.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""Python native tests.""" - -import numpy as np - -import openfl.native as fx - - -def one_hot(labels, classes): - """ - One Hot encode a vector. - - Args: - labels (list): List of labels to onehot encode - classes (int): Total number of categorical classes - - Returns: - np.array: Matrix of one-hot encoded labels - """ - return np.eye(classes)[labels] - - -def build_model(input_shape, - num_classes, - conv_kernel_size=(4, 4), - conv_strides=(2, 2), - conv1_channels_out=16, - conv2_channels_out=32, - final_dense_inputsize=100, - **kwargs): - """ - Define the model architecture. - - Args: - input_shape (numpy.ndarray): The shape of the data - num_classes (int): The number of classes of the dataset - - Returns: - tensorflow.python.keras.engine.sequential.Sequential: The model defined in Keras - - """ - import tensorflow as tf # NOQA - import tensorflow.keras as ke # NOQA - - from tensorflow.keras import Sequential # NOQA - from tensorflow.keras.layers import Conv2D, Flatten, Dense # NOQA - config = tf.compat.v1.ConfigProto() - config.gpu_options.allow_growth = True - config.intra_op_parallelism_threads = 112 - config.inter_op_parallelism_threads = 1 - sess = tf.compat.v1.Session(config=config) - model = Sequential() - - model.add(Conv2D(conv1_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu', - input_shape=input_shape)) - - model.add(Conv2D(conv2_channels_out, - kernel_size=conv_kernel_size, - strides=conv_strides, - activation='relu')) - - model.add(Flatten()) - - model.add(Dense(final_dense_inputsize, activation='relu')) - - model.add(Dense(num_classes, activation='softmax')) - - model.compile(loss=ke.losses.categorical_crossentropy, - optimizer=ke.optimizers.Adam(), - metrics=['accuracy']) - - # initialize the optimizer variables - opt_vars = model.optimizer.variables() - - for v in opt_vars: - v.initializer.run(session=sess) - - return model - - -if __name__ == '__main__': - fx.init('keras_cnn_mnist') - from openfl.federated import FederatedDataSet - from openfl.federated import FederatedModel - from tensorflow.python.keras.utils.data_utils import get_file - - origin_folder = 'https://storage.googleapis.com/tensorflow/tf-keras-datasets/' - path = get_file('mnist.npz', - origin=origin_folder + 'mnist.npz', - file_hash='731c5ac602752760c8e48fbffcf8c3b850d9dc2a2aedcf2cc48468fc17b673d1') - - with np.load(path) as f: - # get all of mnist - X_train = f['x_train'] - y_train = f['y_train'] - - X_valid = f['x_test'] - y_valid = f['y_test'] - img_rows, img_cols = 28, 28 - X_train = X_train.reshape(X_train.shape[0], img_rows, img_cols, 1) - X_valid = X_valid.reshape(X_valid.shape[0], img_rows, img_cols, 1) - X_train = X_train.astype('float32') - X_valid = X_valid.astype('float32') - X_train /= 255 - X_valid /= 255 - - classes = 10 - y_train = one_hot(y_train, classes) - y_valid = one_hot(y_valid, classes) - - feature_shape = X_train.shape[1] - - fl_data = FederatedDataSet(X_train, y_train, X_valid, y_valid, - batch_size=32, num_classes=classes) - fl_model = FederatedModel(build_model=build_model, data_loader=fl_data) - collaborator_models = fl_model.setup(num_collaborators=2) - collaborators = {'one': collaborator_models[0], 'two': collaborator_models[1]} - print(f'Original training data size: {len(X_train)}') - print(f'Original validation data size: {len(X_valid)}\n') - - # Collaborator one's data - print(f'Collaborator one\'s training data size: ' - f'{len(collaborator_models[0].data_loader.X_train)}') - print(f'Collaborator one\'s validation data size: ' - f'{len(collaborator_models[0].data_loader.X_valid)}\n') - - # Collaborator two's data - print(f'Collaborator two\'s training data size: ' - f'{len(collaborator_models[1].data_loader.X_train)}') - print(f'Collaborator two\'s validation data size: ' - f'{len(collaborator_models[1].data_loader.X_valid)}\n') - - print(fx.get_plan()) - final_fl_model = fx.run_experiment(collaborators, {'aggregator.settings.rounds_to_train': 5}) - final_fl_model.save_native('final_pytorch_model.h5') diff --git a/tests/github/python_native_torch.py b/tests/github/python_native_torch.py deleted file mode 100644 index 402110c9ea..0000000000 --- a/tests/github/python_native_torch.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""Python native tests.""" - -import numpy as np - -import openfl.native as fx - - -def one_hot(labels, classes): - """One-hot encode `labels` using `classes` classes.""" - return np.eye(classes)[labels] - - -fx.init('torch_cnn_mnist') - -if __name__ == '__main__': - import torch - import torch.nn as nn - import torch.nn.functional as F - import torch.optim as optim - from torchvision import datasets - from torchvision import transforms - - from openfl.federated import FederatedDataSet - from openfl.federated import FederatedModel - - def cross_entropy(output, target): - """Binary cross-entropy metric.""" - return F.cross_entropy(input=output, target=target) - - class Net(nn.Module): - """PyTorch Neural Network.""" - - def __init__(self): - """Initialize.""" - super(Net, self).__init__() - self.conv1 = nn.Conv2d(1, 16, 3) - self.pool = nn.MaxPool2d(2, 2) - self.conv2 = nn.Conv2d(16, 32, 3) - self.fc1 = nn.Linear(32 * 5 * 5, 32) - self.fc2 = nn.Linear(32, 84) - self.fc3 = nn.Linear(84, 10) - - def forward(self, x): - """Forward pass of the network.""" - x = self.pool(F.relu(self.conv1(x))) - x = self.pool(F.relu(self.conv2(x))) - x = x.view(x.size(0), -1) - x = F.relu(self.fc1(x)) - x = F.relu(self.fc2(x)) - x = self.fc3(x) - return x - - transform = transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) - - trainset = datasets.MNIST(root='./data', train=True, - download=True, transform=transform) - - train_images, train_labels = trainset.train_data, np.array(trainset.train_labels) - train_images = torch.from_numpy(np.expand_dims(train_images, axis=1)).float() - - validset = datasets.MNIST(root='./data', train=False, - download=True, transform=transform) - - valid_images, valid_labels = validset.test_data, np.array(validset.test_labels) - valid_images = torch.from_numpy(np.expand_dims(valid_images, axis=1)).float() - valid_labels = one_hot(valid_labels, 10) - feature_shape = train_images.shape[1] - classes = 10 - - fl_data = FederatedDataSet(train_images, train_labels, valid_images, valid_labels, - batch_size=32, num_classes=classes) - fl_model = FederatedModel(build_model=Net, optimizer=lambda x: optim.Adam(x, lr=1e-4), - loss_fn=cross_entropy, data_loader=fl_data) - collaborator_models = fl_model.setup(num_collaborators=2) - collaborators = {'one': collaborator_models[0], 'two': collaborator_models[1]} - print(f'Original training data size: {len(train_images)}') - print(f'Original validation data size: {len(valid_images)}\n') - - # Collaborator one's data - print(f'Collaborator one\'s training data size: ' - f'{len(collaborator_models[0].data_loader.X_train)}') - print(f'Collaborator one\'s validation data size: ' - f'{len(collaborator_models[0].data_loader.X_valid)}\n') - - # Collaborator two's data - print(f'Collaborator two\'s training data size: ' - f'{len(collaborator_models[1].data_loader.X_train)}') - print(f'Collaborator two\'s validation data size: ' - f'{len(collaborator_models[1].data_loader.X_valid)}\n') - - print(fx.get_plan()) - final_fl_model = fx.run_experiment(collaborators, {'aggregator.settings.rounds_to_train': 5}) - final_fl_model.save_native('final_pytorch_model') diff --git a/tests/openfl/native/__init__.py b/tests/openfl/native/__init__.py deleted file mode 100644 index 319114d31f..0000000000 --- a/tests/openfl/native/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -"""tests.openfl.native package.""" diff --git a/tests/openfl/native/base_example.yaml b/tests/openfl/native/base_example.yaml deleted file mode 100644 index bd0d342898..0000000000 --- a/tests/openfl/native/base_example.yaml +++ /dev/null @@ -1,8 +0,0 @@ -Planet: - Earth: - Continent: - North-America: - USA: - Oregon: 'Portland' - Mars: ['Water', 'Ice'] - Pluto: [] \ No newline at end of file diff --git a/tests/openfl/native/test_update_plan.py b/tests/openfl/native/test_update_plan.py deleted file mode 100644 index e92c2a5cc0..0000000000 --- a/tests/openfl/native/test_update_plan.py +++ /dev/null @@ -1,104 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 -"""Update plan test module.""" -import pytest -from pathlib import Path - -from openfl.federated import Plan -from openfl.native import update_plan - - -@pytest.mark.parametrize( - 'override_config,expected_result', [ - ({}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}), - ({'Planet.Earth.Continent.Australia': 'Sydney'}, - {'Planet': {'Earth': {'Continent': {'Australia': 'Sydney', - 'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}) - ]) -def test_update_plan_new_key_value_addition(override_config, expected_result): - """Test update_plan for adding a new key value pair.""" - plan = Plan() - plan.config = Plan.load(Path('./tests/openfl/native/base_example.yaml')) - result = update_plan(override_config, plan=plan, resolve=False) - assert result.config == expected_result - - -@pytest.mark.parametrize( - 'override_config,expected_result', [ - ({'Planet.Jupiter': ['Sun', 'Rings']}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': [], - 'Jupiter': ['Sun', 'Rings']}}), - ({'Planet.Earth.Continent.Australia': ['Sydney', 'Melbourne']}, - {'Planet': {'Earth': {'Continent': {'Australia': ['Sydney', 'Melbourne'], - 'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}) - ]) -def test_update_plan_new_key_list_value_addition(override_config, expected_result): - """Test update_plan or adding a new key with value as a list.""" - plan = Plan() - plan.config = Plan.load(Path('./tests/openfl/native/base_example.yaml')) - result = update_plan(override_config, plan=plan, resolve=False) - assert result.config == expected_result - - -@pytest.mark.parametrize( - 'override_config,expected_result', [ - ({'Planet.Earth.Continent.North-America.USA.Oregon': 'Salem'}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Salem'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}), - ({'Planet.Mars': 'Moon'}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': 'Moon', - 'Pluto': []}}), - ({'Planet.Pluto': 'Tiny'}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': 'Tiny'}}) - ]) -def test_update_plan_existing_key_value_updation(override_config, expected_result): - """Test update_plan for adding a new key value pair.""" - plan = Plan() - plan.config = Plan.load(Path('./tests/openfl/native/base_example.yaml')) - result = update_plan(override_config, plan=plan, resolve=False) - assert result.config == expected_result - - -@pytest.mark.parametrize( - 'override_config,expected_result', [ - ({'Planet.Mars': ['Water', 'Moon', 'Ice']}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Moon', 'Ice'], - 'Pluto': []}}), - ({'Planet.Mars': ['Water']}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water'], - 'Pluto': []}}), - ({'Planet.Earth.Continent.North-America.USA.Oregon': ['Portland', 'Salem']}, - {'Planet': {'Earth': {'Continent': {'North-America': - {'USA': {'Oregon': ['Portland', 'Salem']}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}), - ({'Planet.Earth.Continent.North-America.USA.Oregon': ['Salem']}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': ['Salem']}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': []}}), - ({'Planet.Pluto': ['Tiny', 'Far']}, - {'Planet': {'Earth': {'Continent': {'North-America': {'USA': {'Oregon': 'Portland'}}}}, - 'Mars': ['Water', 'Ice'], - 'Pluto': ['Tiny', 'Far']}}) - ]) -def test_update_plan_existing_key_list_value_updation(override_config, expected_result): - """Test update_plan or adding a new key with value as a list.""" - plan = Plan() - plan.config = Plan.load(Path('./tests/openfl/native/base_example.yaml')) - result = update_plan(override_config, plan=plan, resolve=False) - assert result.config == expected_result