From 36eb11c0cae940f6bbd834ddff664f8a71748a16 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Wed, 20 Sep 2023 12:35:07 +0200
Subject: [PATCH 01/60] Add FDS how-to guides (#2332)
---
.../how-to-disable-enable-progress-bar.rst | 16 ++++
.../source/how-to-install-flwr-datasets.rst | 46 ++++++++++++
datasets/doc/source/how-to-use-with-numpy.rst | 61 +++++++++++++++
.../doc/source/how-to-use-with-pytorch.rst | 67 +++++++++++++++++
.../doc/source/how-to-use-with-tensorflow.rst | 74 +++++++++++++++++++
5 files changed, 264 insertions(+)
create mode 100644 datasets/doc/source/how-to-disable-enable-progress-bar.rst
create mode 100644 datasets/doc/source/how-to-install-flwr-datasets.rst
create mode 100644 datasets/doc/source/how-to-use-with-numpy.rst
create mode 100644 datasets/doc/source/how-to-use-with-pytorch.rst
create mode 100644 datasets/doc/source/how-to-use-with-tensorflow.rst
diff --git a/datasets/doc/source/how-to-disable-enable-progress-bar.rst b/datasets/doc/source/how-to-disable-enable-progress-bar.rst
new file mode 100644
index 000000000000..95a9c7a562b1
--- /dev/null
+++ b/datasets/doc/source/how-to-disable-enable-progress-bar.rst
@@ -0,0 +1,16 @@
+Disable/Enable Progress Bar
+===========================
+
+You will see a progress bar by default when you download a dataset or apply a map function. Here is how you control
+this behavior.
+
+Disable::
+
+ from datasets.utils.logging import disable_progress_bar
+ disable_progress_bar()
+
+Enable::
+
+ from datasets.utils.logging import enable_progress_bar
+ enable_progress_bar()
+
diff --git a/datasets/doc/source/how-to-install-flwr-datasets.rst b/datasets/doc/source/how-to-install-flwr-datasets.rst
new file mode 100644
index 000000000000..d2fd7923a817
--- /dev/null
+++ b/datasets/doc/source/how-to-install-flwr-datasets.rst
@@ -0,0 +1,46 @@
+Installation
+============
+
+Python Version
+--------------
+
+Flower Datasets requires `Python 3.8 `_ or above.
+
+
+Install stable release (pip)
+----------------------------
+
+Stable releases are available on `PyPI `_
+
+.. code-block:: bash
+
+ python -m pip install flwr-datasets
+
+For vision datasets (e.g. MNIST, CIFAR10) ``flwr-datasets`` should be installed with the ``vision`` extra
+
+.. code-block:: bash
+
+ python -m pip install flwr_datasets[vision]
+
+For audio datasets (e.g. Speech Command) ``flwr-datasets`` should be installed with the ``audio`` extra
+
+.. code-block:: bash
+
+ python -m pip install flwr_datasets[audio]
+
+
+Verify installation
+-------------------
+
+The following command can be used to verify if Flower Datasets was successfully installed:
+
+.. code-block:: bash
+
+ python -c "import flwr_datasets;print(flwr_datasets.__version__)"
+
+If everything worked, it should print the version of Flower Datasets to the command line:
+
+.. code-block:: none
+
+ 0.0.1
+
diff --git a/datasets/doc/source/how-to-use-with-numpy.rst b/datasets/doc/source/how-to-use-with-numpy.rst
new file mode 100644
index 000000000000..c3fbf85969e3
--- /dev/null
+++ b/datasets/doc/source/how-to-use-with-numpy.rst
@@ -0,0 +1,61 @@
+Use with NumPy
+==============
+
+Let's integrate ``flwr-datasets`` with NumPy.
+
+Prepare the desired partitioning::
+
+ from flwr_datasets import FederatedDataset
+
+ fds = FederatedDataset(dataset="cifar10", partitioners={"train": 10})
+ partition = fds.load_partition(0, "train")
+ centralized_dataset = fds.load_full("test")
+
+Transform to NumPy::
+
+ partition_np = partition.with_format("numpy")
+ X_train, y_train = partition_np["img"], partition_np["label"]
+
+That's all. Let's check the dimensions and data types of our ``X_train`` and ``y_train``::
+
+ print(f"The shape of X_train is: {X_train.shape}, dtype: {X_train.dtype}.")
+ print(f"The shape of y_train is: {y_train.shape}, dtype: {y_train.dtype}.")
+
+You should see::
+
+ The shape of X_train is: (500, 32, 32, 3), dtype: uint8.
+ The shape of y_train is: (500,), dtype: int64.
+
+Note that the ``X_train`` values are of type ``uint8``. It is not a problem for the TensorFlow model when passing the
+data as input, but it might remind us to normalize the data - global normalization, pre-channel normalization, or simply
+rescale the data to [0, 1] range::
+
+ X_train = (X_train - X_train.mean()) / X_train.std() # Global normalization
+
+
+CNN Keras model
+---------------
+Here's a quick example of how you can use that data with a simple CNN model::
+
+ import tensorflow as tf
+ from tensorflow.keras import datasets, layers, models
+
+ model = models.Sequential([
+ layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
+ layers.MaxPooling2D(2, 2),
+ layers.Conv2D(64, (3, 3), activation='relu'),
+ layers.MaxPooling2D(2, 2),
+ layers.Conv2D(64, (3, 3), activation='relu'),
+ layers.Flatten(),
+ layers.Dense(64, activation='relu'),
+ layers.Dense(10, activation='softmax')
+ ])
+
+ model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
+ metrics=['accuracy'])
+ model.fit(X_train, y_train, epochs=20, batch_size=64)
+
+You should see about 98% accuracy on the training data at the end of the training.
+
+Note that we used ``"sparse_categorical_crossentropy"``. Make sure to keep it that way if you don't want to one-hot-encode
+the labels.
diff --git a/datasets/doc/source/how-to-use-with-pytorch.rst b/datasets/doc/source/how-to-use-with-pytorch.rst
new file mode 100644
index 000000000000..5981f88c26b8
--- /dev/null
+++ b/datasets/doc/source/how-to-use-with-pytorch.rst
@@ -0,0 +1,67 @@
+Use with PyTorch
+================
+Let's integrate ``flwr-datasets`` with PyTorch DataLoaders and keep your PyTorch Transform applied to the data.
+
+Standard setup - download the dataset, choose the partitioning::
+
+ from flwr_datasets import FederatedDataset
+
+ fds = FederatedDataset(dataset="cifar10", partitioners={"train": 10})
+ partition = fds.load_partition(0, "train")
+ centralized_dataset = fds.load_full("test")
+
+Determine the names of our features (you can alternatively do that directly on the Hugging Face website). The name can
+vary e.g. "img" or "image", "label" or "labels"::
+
+ partition.features
+
+In case of CIFAR10, you should see the following output
+
+.. code-block:: none
+
+ {'img': Image(decode=True, id=None),
+ 'label': ClassLabel(names=['airplane', 'automobile', 'bird', 'cat', 'deer', 'dog',
+ 'frog', 'horse', 'ship', 'truck'], id=None)}
+
+Apply Transforms, Create DataLoader. We will use the `map() `_
+function. Please note that the map will modify the existing dataset if the key in the dictionary you return is already present
+and append a new feature if it did not exist before. Below, we modify the "img" feature of our dataset.::
+
+ from torch.utils.data import DataLoader
+ from torchvision.transforms import ToTensor
+
+ transforms = ToTensor()
+ partition_torch = partition.map(
+ lambda img: {"img": transforms(img)}, input_columns="img"
+ ).with_format("torch")
+ dataloader = DataLoader(partition_torch, batch_size=64)
+
+We advise you to keep the
+`ToTensor() `_ transform (especially if
+you used it in your PyTorch code) because it swaps the dimensions from (H x W x C) to (C x H x W). This order is
+expected by a model with a convolutional layer.
+
+If you want to divide the dataset, you can use (at any point before passing the dataset to the DataLoader)::
+
+ partition_train_test = partition.train_test_split(test_size=0.2)
+ partition_train = partition_train_test["train"]
+ partition_test = partition_train_test["test"]
+
+Or you can simply calculate the indices yourself::
+
+ partition_len = len(partition)
+ partition_train = partition[:int(0.8 * partition_len)]
+ partition_test = partition[int(0.8 * partition_len):]
+
+And during the training loop, you need to apply one change. With a typical dataloader, you get a list returned for each iteration::
+
+ for batch in all_from_pytorch_dataloader:
+ images, labels = batch
+ # Or alternatively:
+ # images, labels = batch[0], batch[1]
+
+With this dataset, you get a dictionary, and you access the data a little bit differently (via keys not by index)::
+
+ for batch in dataloader:
+ images, labels = batch["img"], batch["label"]
+
diff --git a/datasets/doc/source/how-to-use-with-tensorflow.rst b/datasets/doc/source/how-to-use-with-tensorflow.rst
new file mode 100644
index 000000000000..86a1f4e0da8a
--- /dev/null
+++ b/datasets/doc/source/how-to-use-with-tensorflow.rst
@@ -0,0 +1,74 @@
+Use with TensorFlow
+===================
+
+Let's integrate ``flwr-datasets`` with TensorFlow. We show you three ways how to convert the data into the formats
+that ``TensorFlow``'s models expect. Please note that, especially for the smaller datasets, the performance of the
+following methods is very close. We recommend you choose the method you are the most comfortable with.
+
+NumPy
+-----
+The first way is to transform the data into the NumPy arrays. It's an easier option that is commonly used. Feel free to
+follow the :doc:`how-to-use-with-numpy` tutorial, especially if you are a beginner.
+
+.. _tensorflow-dataset:
+
+TensorFlow Dataset
+------------------
+Work with ``TensorFlow Dataset`` abstraction.
+
+Standard setup::
+
+ from flwr_datasets import FederatedDataset
+
+ fds = FederatedDataset(dataset="cifar10", partitioners={"train": 10})
+ partition = fds.load_partition(0, "train")
+ centralized_dataset = fds.load_full("test")
+
+Transformation to the TensorFlow Dataset::
+
+ tf_dataset = partition.to_tf_dataset(columns="img", label_cols="label", batch_size=64,
+ shuffle=True)
+ # Assuming you have defined your model and compiled it
+ model.fit(tf_dataset, epochs=20)
+
+TensorFlow Tensors
+------------------
+Change the data type to TensorFlow Tensors (it's not the TensorFlow dataset).
+
+Standard setup::
+
+ from flwr_datasets import FederatedDataset
+
+ fds = FederatedDataset(dataset="cifar10", partitioners={"train": 10})
+ partition = fds.load_partition(0, "train")
+ centralized_dataset = fds.load_full("test")
+
+Transformation to the TensorFlow Tensors ::
+
+ data_tf = partition.with_format("tf")
+ # Assuming you have defined your model and compiled it
+ model.fit(data_tf["img"], data_tf["label"], epochs=20, batch_size=64)
+
+CNN Keras Model
+---------------
+Here's a quick example of how you can use that data with a simple CNN model (it assumes you created the TensorFlow
+dataset as in the section above, see :ref:`TensorFlow Dataset `)::
+
+ import tensorflow as tf
+ from tensorflow.keras import datasets, layers, models
+
+ model = models.Sequential([
+ layers.Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
+ layers.MaxPooling2D(2, 2),
+ layers.Conv2D(64, (3, 3), activation='relu'),
+ layers.MaxPooling2D(2, 2),
+ layers.Conv2D(64, (3, 3), activation='relu'),
+ layers.Flatten(),
+ layers.Dense(64, activation='relu'),
+ layers.Dense(10, activation='softmax')
+ ])
+
+ model.compile(optimizer='adam', loss='sparse_categorical_crossentropy',
+ metrics=['accuracy'])
+ model.fit(tf_dataset, epochs=20)
+
From d434595e315fc204b9d48c2ff471cb9ceb5f5f64 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Thu, 21 Sep 2023 11:18:05 +0200
Subject: [PATCH 02/60] Add paths specification to CI triggers for FDS (#2399)
---------
Co-authored-by: Taner Topal
---
.github/workflows/datasets.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/.github/workflows/datasets.yml b/.github/workflows/datasets.yml
index 60eb9e49db45..47e9f2aed926 100644
--- a/.github/workflows/datasets.yml
+++ b/.github/workflows/datasets.yml
@@ -4,9 +4,13 @@ on:
push:
branches:
- main
+ paths:
+ - "datasets/**"
pull_request:
branches:
- main
+ paths:
+ - "datasets/**"
concurrency:
group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
From a43044c3476a5d6178afc6a615ad36264239ba2a Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Thu, 21 Sep 2023 22:52:04 +0200
Subject: [PATCH 03/60] Add TensorFlow integration tests with FDS (#2350)
---
datasets/e2e/tensorflow/pyproject.toml | 15 +++
datasets/e2e/tensorflow/tensorflow_test.py | 102 +++++++++++++++++++++
2 files changed, 117 insertions(+)
create mode 100644 datasets/e2e/tensorflow/pyproject.toml
create mode 100644 datasets/e2e/tensorflow/tensorflow_test.py
diff --git a/datasets/e2e/tensorflow/pyproject.toml b/datasets/e2e/tensorflow/pyproject.toml
new file mode 100644
index 000000000000..9c5c72c46400
--- /dev/null
+++ b/datasets/e2e/tensorflow/pyproject.toml
@@ -0,0 +1,15 @@
+[build-system]
+requires = ["poetry-core>=1.4.0"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.poetry]
+name = "fds-e2-tensorflow"
+version = "0.1.0"
+description = "Flower Datasets with TensorFlow"
+authors = ["The Flower Authors "]
+
+[tool.poetry.dependencies]
+python = "^3.8"
+flwr-datasets = { path = "./../../", extras = ["vision"] }
+tensorflow-cpu = "^2.9.1, !=2.11.1"
+parameterized = "==0.9.0"
diff --git a/datasets/e2e/tensorflow/tensorflow_test.py b/datasets/e2e/tensorflow/tensorflow_test.py
new file mode 100644
index 000000000000..e041bcb8f8cc
--- /dev/null
+++ b/datasets/e2e/tensorflow/tensorflow_test.py
@@ -0,0 +1,102 @@
+import unittest
+
+import numpy as np
+import tensorflow as tf
+from datasets.utils.logging import disable_progress_bar
+from parameterized import parameterized_class, parameterized
+from tensorflow.keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
+from tensorflow.keras.models import Sequential
+
+from flwr_datasets import FederatedDataset
+
+
+def SimpleCNN():
+ model = Sequential([
+ Conv2D(32, (3, 3), activation='relu', input_shape=(32, 32, 3)),
+ MaxPooling2D(2, 2),
+ Conv2D(64, (3, 3), activation='relu'),
+ MaxPooling2D(2, 2),
+ Flatten(),
+ Dense(64, activation='relu'),
+ Dense(10, activation='softmax')
+ ])
+ return model
+
+
+@parameterized_class(
+ [
+ {"dataset_name": "cifar10", "test_split": "test"},
+ {"dataset_name": "cifar10", "test_split": "test"},
+ ]
+)
+class FdsToTensorFlow(unittest.TestCase):
+ """Test the conversion from FDS to PyTorch Dataset and Dataloader."""
+
+ dataset_name = ""
+ test_split = ""
+ expected_img_shape_after_transform = [32, 32, 3]
+
+ @classmethod
+ def setUpClass(cls):
+ """Disable progress bar to keep the log clean.
+ """
+ disable_progress_bar()
+
+ def _create_tensorflow_dataset(self, batch_size: int) -> tf.data.Dataset:
+ """Create a tensorflow dataset from the FederatedDataset."""
+ partition_id = 0
+ fds = FederatedDataset(dataset=self.dataset_name, partitioners={"train": 100})
+ partition = fds.load_partition(partition_id, "train")
+ tf_dataset = partition.to_tf_dataset(columns="img", label_cols="label",
+ batch_size=batch_size,
+ shuffle=False)
+ return tf_dataset
+
+ def test_create_partition_dataset_shape(self) -> None:
+ """Test if the DataLoader returns batches with the expected shape."""
+ batch_size = 16
+ dataset = self._create_tensorflow_dataset(batch_size)
+ batch = next(iter(dataset))
+ images = batch[0]
+ self.assertEqual(tuple(images.shape),
+ (batch_size, *self.expected_img_shape_after_transform))
+
+ def test_create_partition_dataloader_with_transforms_batch_type(self) -> None:
+ """Test if the DataLoader returns batches of type dictionary."""
+ batch_size = 16
+ dataset = self._create_tensorflow_dataset(batch_size)
+ batch = next(iter(dataset))
+ self.assertIsInstance(batch, tuple)
+
+ def test_create_partition_dataloader_with_transforms_data_type(self) -> None:
+ """Test to verify if the data in the DataLoader batches are of type Tensor."""
+ batch_size = 16
+ dataset = self._create_tensorflow_dataset(batch_size)
+ batch = next(iter(dataset))
+ images = batch[0]
+ self.assertIsInstance(images, tf.Tensor)
+
+ @parameterized.expand([
+ ("not_nan", np.isnan),
+ ("not_inf", np.isinf),
+ ])
+ def test_train_model_loss_value(self, name, condition_func):
+ model = SimpleCNN()
+ model.compile(optimizer='adam',
+ loss='sparse_categorical_crossentropy',
+ metrics=['accuracy'])
+
+ dataset = self._create_tensorflow_dataset(16)
+
+ # Perform a single epoch of training
+ history = model.fit(dataset, epochs=1, verbose=0)
+
+ # Fetch the last loss from history
+ last_loss = history.history['loss'][-1]
+
+ # Check if the last loss is NaN or Infinity
+ self.assertFalse(condition_func(last_loss))
+
+
+if __name__ == '__main__':
+ unittest.main()
From b569d2a2c4025735d5209ffd0244771e83b4e0f7 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Thu, 21 Sep 2023 22:59:36 +0200
Subject: [PATCH 04/60] Update FDS docs index (#2337)
---
datasets/doc/source/index.rst | 97 +++++++++++++++++++--
datasets/doc/source/tutorial-quickstart.rst | 2 +-
2 files changed, 89 insertions(+), 10 deletions(-)
diff --git a/datasets/doc/source/index.rst b/datasets/doc/source/index.rst
index 81a08286b6fd..7b19624b341a 100644
--- a/datasets/doc/source/index.rst
+++ b/datasets/doc/source/index.rst
@@ -1,8 +1,93 @@
-Flower Datasets Documentation
-=============================
+Flower Datasets
+===============
+
+Flower Datasets (``flwr-datasets``) is a library to quickly and easily create datasets for federated
+learning/analytics/evaluation. It is created by the ``Flower Labs`` team that also created `Flower `_ - a Friendly Federated Learning Framework.
+
+Flower Datasets Framework
+-------------------------
+
+Tutorials
+~~~~~~~~~
+
+A learning-oriented series of tutorials is the best place to start.
+
+.. toctree::
+ :maxdepth: 1
+ :caption: Tutorial
+
+ tutorial-quickstart
+
+How-to guides
+~~~~~~~~~~~~~
+
+Problem-oriented how-to guides show step-by-step how to achieve a specific goal.
+
+.. toctree::
+ :maxdepth: 1
+ :caption: How-to guides
+
+ how-to-install-flwr-datasets
+ how-to-use-with-pytorch
+ how-to-use-with-tensorflow
+ how-to-use-with-numpy
+ how-to-disable-enable-progress-bar
+
+References
+~~~~~~~~~~
+
+Information-oriented API reference and other reference material.
+
+.. toctree::
+ :maxdepth: 2
+ :caption: API reference
+
+ ref-api-flwr-datasets
+
+Main features
+-------------
+Flower Datasets library supports:
+
+- **downloading datasets** - choose the dataset from Hugging Face's ``dataset``
+- **partitioning datasets** - customize the partitioning scheme
+- **creating centralized datasets** - leave parts of the dataset unpartitioned (e.g. for centralized evaluation)
-Welcome to Flower Datasets' documentation. `Flower `_ is a friendly federated learning framework.
+Thanks to using Hugging Face's ``datasets`` used under the hood, Flower Datasets integrates with the following popular formats/frameworks:
+- Hugging Face
+- PyTorch
+- TensorFlow
+- Numpy
+- Pandas
+- Jax
+- Arrow
+
+Install
+-------
+
+The simplest install is
+
+.. code-block:: bash
+
+ python -m pip install flwr-datasets
+
+If you plan to use the image datasets
+
+.. code-block:: bash
+
+ python -m pip install flwr-datasets[vision]
+
+If you plan to use the audio datasets
+
+.. code-block:: bash
+
+ python -m pip install flwr-datasets[audio]
+
+Check out the full details on the download in :doc:`how-to-install-flwr-datasets`.
+
+How To Use the library
+----------------------
+Learn how to use the ``flwr-datasets`` library from the :doc:`tutorial-quickstart` examples .
Join the Flower Community
-------------------------
@@ -14,9 +99,3 @@ The Flower Community is growing quickly - we're a friendly group of researchers,
:shadow:
Join us on Slack
-
-
-Flower Datasets
----------------
-
-
diff --git a/datasets/doc/source/tutorial-quickstart.rst b/datasets/doc/source/tutorial-quickstart.rst
index 69d42f16a3b6..d1992b8e68fe 100644
--- a/datasets/doc/source/tutorial-quickstart.rst
+++ b/datasets/doc/source/tutorial-quickstart.rst
@@ -41,7 +41,7 @@ supported by your framework.
Conversion
----------
-For more detailed instructions, go to :doc:`how-to`.
+For more detailed instructions, go to :doc:`how-to-use-with-pytorch`.
PyTorch DataLoader
^^^^^^^^^^^^^^^^^^
From b63b775b4c35868d33f2b30dc123cc87773a6fc9 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Fri, 22 Sep 2023 16:29:53 +0200
Subject: [PATCH 05/60] Fix default contiguous value in IidPartitioner (#2406)
---------
Co-authored-by: Taner Topal
---
.../flwr_datasets/partitioner/iid_partitioner.py | 2 +-
.../partitioner/iid_partitioner_test.py | 12 +++++++++---
2 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/datasets/flwr_datasets/partitioner/iid_partitioner.py b/datasets/flwr_datasets/partitioner/iid_partitioner.py
index c8dbf8294fec..37b97468cadf 100644
--- a/datasets/flwr_datasets/partitioner/iid_partitioner.py
+++ b/datasets/flwr_datasets/partitioner/iid_partitioner.py
@@ -48,5 +48,5 @@ def load_partition(self, idx: int) -> datasets.Dataset:
single dataset partition
"""
return self.dataset.shard(
- num_shards=self._num_partitions, index=idx, contiguous=True
+ num_shards=self._num_partitions, index=idx, contiguous=False
)
diff --git a/datasets/flwr_datasets/partitioner/iid_partitioner_test.py b/datasets/flwr_datasets/partitioner/iid_partitioner_test.py
index d89eefeba9f2..5f851807f4bd 100644
--- a/datasets/flwr_datasets/partitioner/iid_partitioner_test.py
+++ b/datasets/flwr_datasets/partitioner/iid_partitioner_test.py
@@ -18,6 +18,7 @@
import unittest
from typing import Tuple
+import numpy as np
from parameterized import parameterized
from datasets import Dataset
@@ -100,11 +101,16 @@ def test_load_partition_correct_data(
self, num_partitions: int, num_rows: int
) -> None:
"""Test if the data in partition is equal to the expected."""
- _, partitioner = _dummy_setup(num_partitions, num_rows)
- partition_size = num_rows // num_partitions
+ dataset, partitioner = _dummy_setup(num_partitions, num_rows)
partition_index = 2
partition = partitioner.load_partition(partition_index)
- self.assertEqual(partition["features"][0], partition_index * partition_size)
+ row_id = 0
+ self.assertEqual(
+ partition["features"][row_id],
+ dataset[np.arange(partition_index, len(dataset), num_partitions)][
+ "features"
+ ][row_id],
+ )
@parameterized.expand( # type: ignore
[
From 2b9da50283b70075b69c1dca22a92ac99a7b35da Mon Sep 17 00:00:00 2001
From: Taner Topal
Date: Fri, 22 Sep 2023 11:11:29 -0700
Subject: [PATCH 06/60] Fix repository URL for datasets (#2415)
---
datasets/pyproject.toml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/datasets/pyproject.toml b/datasets/pyproject.toml
index 6067ff0517db..954441e5d2e4 100644
--- a/datasets/pyproject.toml
+++ b/datasets/pyproject.toml
@@ -10,7 +10,7 @@ license = "Apache-2.0"
authors = ["The Flower Authors "]
readme = "README.md"
homepage = "https://flower.dev"
-repository = "https://github.com/adap/flower/datasets"
+repository = "https://github.com/adap/flower"
documentation = "https://flower.dev/docs/datasets"
keywords = [
"flower",
From 725176452291d4abe16549d6777e2f7b73e43b80 Mon Sep 17 00:00:00 2001
From: Navin Chandra <98466550+navin772@users.noreply.github.com>
Date: Sat, 23 Sep 2023 02:09:40 +0530
Subject: [PATCH 07/60] Fix video link in colab notebook of
flower-in-30-minutes (#2414)
---
examples/flower-in-30-minutes/tutorial.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/flower-in-30-minutes/tutorial.ipynb b/examples/flower-in-30-minutes/tutorial.ipynb
index b1686529c462..336ec4c19644 100644
--- a/examples/flower-in-30-minutes/tutorial.ipynb
+++ b/examples/flower-in-30-minutes/tutorial.ipynb
@@ -23,7 +23,7 @@
"## Complementary Content\n",
"\n",
"But before do so, let me point you to a few video tutorials in the [Flower Youtube channel](https://www.youtube.com/@flowerlabs) that you might want to check out after this tutorial. We post new videos fairly regularly with new content:\n",
- "* **[VIDEO]** quickstart-tensorflow: [15-min video on how to start with Flower + Tensorflow/Keras](https://www.youtube.com/watch?v=jOmmuzMIQ4c)\n",
+ "* **[VIDEO]** quickstart-tensorflow: [15-min video on how to start with Flower + Tensorflow/Keras](https://www.youtube.com/watch?v=FGTc2TQq7VM)\n",
"* **[VIDEO]** quickstart-pytorch: [20-min video on how to start with Flower + PyTorch](https://www.youtube.com/watch?v=jOmmuzMIQ4c)\n",
"* **[VIDEO]** Flower simulation mini-series: [9 line-by-line video tutorials](https://www.youtube.com/watch?v=cRebUIGB5RU&list=PLNG4feLHqCWlnj8a_E1A_n5zr2-8pafTB)"
]
From dca3102c8ffcec1c9f3194422e3a1a6e211f20b1 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Mon, 25 Sep 2023 14:16:47 +0200
Subject: [PATCH 08/60] Fix code block from python to bash (#2405)
---
datasets/doc/source/tutorial-quickstart.rst | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/datasets/doc/source/tutorial-quickstart.rst b/datasets/doc/source/tutorial-quickstart.rst
index d1992b8e68fe..8a70ee8854be 100644
--- a/datasets/doc/source/tutorial-quickstart.rst
+++ b/datasets/doc/source/tutorial-quickstart.rst
@@ -5,17 +5,23 @@ Run Flower Datasets as fast as possible by learning only the essentials.
Install Federated Datasets
--------------------------
-Run on the command line::
+Run on the command line
+
+.. code-block:: bash
python -m pip install flwr-datasets[vision]
Install the ML framework
------------------------
-TensorFlow::
+TensorFlow
+
+.. code-block:: bash
pip install tensorflow
-PyTorch::
+PyTorch
+
+.. code-block:: bash
pip install torch torchvision
From 2fd02c52d881cab7da82ac7c2aea50a02ad33d67 Mon Sep 17 00:00:00 2001
From: Adam Narozniak <51029327+adam-narozniak@users.noreply.github.com>
Date: Mon, 25 Sep 2023 15:40:37 +0200
Subject: [PATCH 09/60] Make Flower Datasets docs publishable (#2418)
---
.github/workflows/docs.yml | 1 +
dev/build-docs.sh | 4 ++++
2 files changed, 5 insertions(+)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 52f9a49a259a..f74532dd721c 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -39,3 +39,4 @@ jobs:
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./doc/build/html/ s3://flower.dev/docs/framework
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./baselines/doc/build/html/ s3://flower.dev/docs/baselines
aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./examples/doc/build/html/ s3://flower.dev/docs/examples
+ aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./datasets/doc/build/html/ s3://flower.dev/docs/datasets
diff --git a/dev/build-docs.sh b/dev/build-docs.sh
index ca57536901b2..c464cf908c87 100755
--- a/dev/build-docs.sh
+++ b/dev/build-docs.sh
@@ -17,3 +17,7 @@ cd examples/doc
make docs
cd $ROOT
+cd datasets/doc
+make docs
+
+cd $ROOT
From 204a4fee6289266b73afdfeae3e63d7ee9125e9c Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Mon, 25 Sep 2023 15:50:06 +0200
Subject: [PATCH 10/60] Add parameter to actions/checkout to fix doc versions
(#2419)
---
.github/workflows/docs.yml | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index f74532dd721c..78b04c5138d4 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -21,6 +21,8 @@ jobs:
name: Build and deploy
steps:
- uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
- name: Bootstrap
uses: ./.github/actions/bootstrap
- name: Install pandoc
From 9ee473152f2fde2a5b05b99829db27a607dc46ec Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Mon, 25 Sep 2023 22:37:06 +0200
Subject: [PATCH 11/60] Add new wheel building job and use it in E2E tests
(#2291)
---
.github/workflows/e2e.yml | 37 +++++++++++++++++++++++++++++++++++++
1 file changed, 37 insertions(+)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 3b70db43a6c8..214c8579d450 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -16,9 +16,39 @@ env:
FLWR_TELEMETRY_ENABLED: 0
jobs:
+ wheel:
+ runs-on: ubuntu-22.04
+ name: Build, test and upload wheel
+ steps:
+ - uses: actions/checkout@v3
+ - name: Bootstrap
+ uses: ./.github/actions/bootstrap
+ - name: Install dependencies (mandatory only)
+ run: python -m poetry install
+ - name: Build wheel
+ run: ./dev/build.sh
+ - name: Test wheel
+ run: ./dev/test-wheel.sh
+ - name: Upload wheel
+ id: upload
+ env:
+ AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
+ run: |
+ cd ./dist
+ echo "WHL_PATH=$(ls *.whl)" >> "$GITHUB_OUTPUT"
+ sha_short=$(git rev-parse --short HEAD)
+ echo "SHORT_SHA=$sha_short" >> "$GITHUB_OUTPUT"
+ aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://artifact.flower.dev/py/${{ github.head_ref }}/$sha_short --recursive
+ outputs:
+ whl_path: ${{ steps.upload.outputs.WHL_PATH }}
+ short_sha: ${{ steps.upload.outputs.SHORT_SHA }}
+
frameworks:
runs-on: ubuntu-22.04
timeout-minutes: 10
+ needs: wheel
# Using approach described here:
# https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
strategy:
@@ -89,6 +119,9 @@ jobs:
python-version: 3.8
- name: Install dependencies
run: python -m poetry install
+ - name: Install Flower wheel from artifact store
+ run: |
+ python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Download dataset
if: ${{ matrix.dataset }}
run: python -c "${{ matrix.dataset }}"
@@ -102,6 +135,7 @@ jobs:
strategies:
runs-on: ubuntu-22.04
timeout-minutes: 10
+ needs: wheel
strategy:
matrix:
strat: ["FedMedian", "FedTrimmedAvg", "QFedAvg", "FaultTolerantFedAvg", "FedAvgM", "FedAdam", "FedAdagrad", "FedYogi"]
@@ -119,6 +153,9 @@ jobs:
- name: Install dependencies
run: |
python -m poetry install
+ - name: Install Flower wheel from artifact store
+ run: |
+ python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Cache Datasets
uses: actions/cache@v3
with:
From 4a8ac498c9cc64178fe3fb1f3b333ffb9057fedd Mon Sep 17 00:00:00 2001
From: Heng Pan <134433891+panh99@users.noreply.github.com>
Date: Thu, 28 Sep 2023 01:43:07 +0100
Subject: [PATCH 12/60] Adapt secaggplus-mt to use workload_id (#2408)
---
examples/secaggplus-mt/driver.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/examples/secaggplus-mt/driver.py b/examples/secaggplus-mt/driver.py
index c168edf070af..4e0a53ed1c91 100644
--- a/examples/secaggplus-mt/driver.py
+++ b/examples/secaggplus-mt/driver.py
@@ -23,7 +23,7 @@ def merge(_task: task_pb2.Task, _merge_task: task_pb2.Task) -> task_pb2.Task:
task_pb2.TaskIns(
task_id="", # Do not set, will be created and set by the DriverAPI
group_id="",
- workload_id="",
+ workload_id=workload_id,
task=merge(
task,
task_pb2.Task(
@@ -84,8 +84,14 @@ def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics:
# -------------------------------------------------------------------------- Driver SDK
driver.connect()
+create_workload_res: driver_pb2.CreateWorkloadResponse = driver.create_workload(
+ req=driver_pb2.CreateWorkloadRequest()
+)
# -------------------------------------------------------------------------- Driver SDK
+workload_id = create_workload_res.workload_id
+print(f"Created workload id {workload_id}")
+
history = History()
for server_round in range(num_rounds):
print(f"Commencing server round {server_round + 1}")
@@ -113,7 +119,7 @@ def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics:
# loop and wait until enough client nodes are available.
while True:
# Get a list of node ID's from the server
- get_nodes_req = driver_pb2.GetNodesRequest()
+ get_nodes_req = driver_pb2.GetNodesRequest(workload_id=workload_id)
# ---------------------------------------------------------------------- Driver SDK
get_nodes_res: driver_pb2.GetNodesResponse = driver.get_nodes(
@@ -121,7 +127,7 @@ def weighted_average(metrics: List[Tuple[int, Metrics]]) -> Metrics:
)
# ---------------------------------------------------------------------- Driver SDK
- all_node_ids: List[int] = get_nodes_res.node_ids
+ all_node_ids: List[int] = [node.node_id for node in get_nodes_res.nodes]
if len(all_node_ids) >= num_client_nodes_per_round:
# Sample client nodes
From e63229cb5e87a7e07cfc18ca44a95e6d4d8e4655 Mon Sep 17 00:00:00 2001
From: Javier
Date: Fri, 29 Sep 2023 14:13:49 +0100
Subject: [PATCH 13/60] Use minimal ray package (#2402)
---
doc/source/ref-changelog.md | 2 +-
pyproject.toml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md
index e1d90b01fb35..ced58029bd5a 100644
--- a/doc/source/ref-changelog.md
+++ b/doc/source/ref-changelog.md
@@ -20,7 +20,7 @@
- **General updates to the simulation engine** ([#2331](https://github.com/adap/flower/pull/2331))
-- **General improvements** ([#2309](https://github.com/adap/flower/pull/2309), [#2310](https://github.com/adap/flower/pull/2310), [2313](https://github.com/adap/flower/pull/2313), [#2316](https://github.com/adap/flower/pull/2316), [2317](https://github.com/adap/flower/pull/2317),[#2349](https://github.com/adap/flower/pull/2349), [#2360](https://github.com/adap/flower/pull/2360))
+- **General improvements** ([#2309](https://github.com/adap/flower/pull/2309), [#2310](https://github.com/adap/flower/pull/2310), [2313](https://github.com/adap/flower/pull/2313), [#2316](https://github.com/adap/flower/pull/2316), [2317](https://github.com/adap/flower/pull/2317),[#2349](https://github.com/adap/flower/pull/2349), [#2360](https://github.com/adap/flower/pull/2360), [#2402](https://github.com/adap/flower/pull/2402))
Flower received many improvements under the hood, too many to list here.
diff --git a/pyproject.toml b/pyproject.toml
index 91d7d810f810..dfdd75ba11ab 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -66,7 +66,7 @@ cryptography = "^41.0.2"
pycryptodome = "^3.18.0"
iterators = "^0.0.2"
# Optional dependencies (VCE)
-ray = { version = "==2.6.3", extras = ["default"], optional = true }
+ray = { version = "==2.6.3", optional = true }
pydantic = { version = "<2.0.0", optional = true }
# Optional dependencies (REST transport layer)
requests = { version = "^2.31.0", optional = true }
From ab7f77584b7caf37e98752016249b0cbfc6abd79 Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Sat, 30 Sep 2023 20:05:04 +0200
Subject: [PATCH 14/60] Make videos bigger to allow for index (#2436)
---
doc/source/index.rst | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 48f8d59ea9b7..4ac99cc24c09 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -61,17 +61,15 @@ A learning-oriented series of federated learning tutorials, the best place to st
QUICKSTART TUTORIALS: :doc:`PyTorch ` | :doc:`TensorFlow ` | :doc:`🤗 Transformers ` | :doc:`JAX ` | :doc:`Pandas ` | :doc:`fastai ` | :doc:`PyTorch Lightning ` | :doc:`MXNet ` | :doc:`scikit-learn ` | :doc:`XGBoost ` | :doc:`Android ` | :doc:`iOS `
-.. grid:: 2
+We also made video tutorials for PyTorch:
- .. grid-item-card:: PyTorch
+.. youtube:: jOmmuzMIQ4c
+ :width: 80%
- .. youtube:: jOmmuzMIQ4c
- :width: 100%
+And TensorFlow:
- .. grid-item-card:: TensorFlow
-
- .. youtube:: FGTc2TQq7VM
- :width: 100%
+.. youtube:: FGTc2TQq7VM
+ :width: 80%
How-to guides
~~~~~~~~~~~~~
From 2997b79b9d331f8b0e83dbcb387f41f47eaf11c7 Mon Sep 17 00:00:00 2001
From: Taner Topal
Date: Wed, 4 Oct 2023 06:06:21 -0700
Subject: [PATCH 15/60] Set AWS_REGION correctly for CI job (#2454)
---
.github/workflows/e2e.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 214c8579d450..74e9b4c2684e 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -32,7 +32,7 @@ jobs:
- name: Upload wheel
id: upload
env:
- AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
+ AWS_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
run: |
From a6105e73a7c8aa7e2964b21629b64c1e2acfc84e Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Wed, 4 Oct 2023 16:22:15 +0200
Subject: [PATCH 16/60] Remove duplicate workflow (#2445)
---
.github/workflows/release.yml | 26 --------------------------
1 file changed, 26 deletions(-)
delete mode 100644 .github/workflows/release.yml
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index d8f4e403482b..000000000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-name: Release
-
-on:
- schedule:
- - cron: "0 23 * * *"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
- cancel-in-progress: true
-
-env:
- FLWR_TELEMETRY_ENABLED: 0
-
-jobs:
- nightly_release:
- runs-on: ubuntu-22.04
- name: Nightly
- steps:
- - uses: actions/checkout@v4
- - name: Bootstrap
- uses: ./.github/actions/bootstrap
- - name: Release nightly
- env:
- PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
- run: |
- ./dev/publish-nightly.sh
From 2a7f62709411f74b8de206a920b5ecf4fb54baec Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Thu, 5 Oct 2023 09:22:11 +0200
Subject: [PATCH 17/60] Revert to AWS_DEFAULT_REGION (#2459)
---
.github/workflows/e2e.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 74e9b4c2684e..214c8579d450 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -32,7 +32,7 @@ jobs:
- name: Upload wheel
id: upload
env:
- AWS_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
+ AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
run: |
From b02d0420281421dbe6413ea82d9e41e8b2a5608a Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Thu, 5 Oct 2023 09:26:38 +0200
Subject: [PATCH 18/60] Update android-kotlin example requirements (#2462)
---
examples/android-kotlin/pyproject.toml | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/examples/android-kotlin/pyproject.toml b/examples/android-kotlin/pyproject.toml
index dee6cbc35711..9cf0688d83b5 100644
--- a/examples/android-kotlin/pyproject.toml
+++ b/examples/android-kotlin/pyproject.toml
@@ -1,3 +1,7 @@
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
[tool.poetry]
name = "flower-android-kotlin"
version = "0.1.0"
@@ -7,7 +11,3 @@ authors = ["Steven Hé (Sīchà ng) "]
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
flwr = ">=1.0,<2.0"
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
From afca860e3f73c74fa90cd40846c0dd6087619ff0 Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Thu, 5 Oct 2023 09:33:06 +0200
Subject: [PATCH 19/60] Update mt-pytorch example requirements (#2463)
---
examples/mt-pytorch/pyproject.toml | 7 +++----
examples/mt-pytorch/requirements.txt | 4 ++--
2 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/examples/mt-pytorch/pyproject.toml b/examples/mt-pytorch/pyproject.toml
index f285af016499..4978035495ea 100644
--- a/examples/mt-pytorch/pyproject.toml
+++ b/examples/mt-pytorch/pyproject.toml
@@ -10,8 +10,7 @@ authors = ["The Flower Authors "]
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
-flwr-nightly = { version = "^1.5.0.dev20230629", extras = ["simulation", "rest"] }
-# flwr = { path = "../../", develop = true, extras = ["simulation", "rest"] }
-torch = "^2.0.1"
-torchvision = "^0.15.2"
+flwr-nightly = {version = ">=1.0,<2.0", extras = ["rest", "simulation"]}
+torch = "1.13.1"
+torchvision = "0.14.1"
tqdm = "4.65.0"
diff --git a/examples/mt-pytorch/requirements.txt b/examples/mt-pytorch/requirements.txt
index 98b7617e776d..ae0a65386f2b 100644
--- a/examples/mt-pytorch/requirements.txt
+++ b/examples/mt-pytorch/requirements.txt
@@ -1,4 +1,4 @@
-flwr-nightly[simulation,rest]
+flwr-nightly[rest,simulation]>=1.0, <2.0
torch==1.13.1
-torchvision==0.13.0
+torchvision==0.14.1
tqdm==4.65.0
From e31d48860aec3345d5a6fb69585803eec908e249 Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Thu, 5 Oct 2023 09:54:17 +0200
Subject: [PATCH 20/60] Update iOS example requirements (#2461)
---
examples/ios/pyproject.toml | 2 +-
examples/ios/requirements.txt | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/examples/ios/pyproject.toml b/examples/ios/pyproject.toml
index 531e9253e0d1..c1bdbb815bd5 100644
--- a/examples/ios/pyproject.toml
+++ b/examples/ios/pyproject.toml
@@ -10,4 +10,4 @@ authors = ["The Flower Authors "]
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
-flwr = "^1.0.0"
+flwr = ">=1.0,<2.0"
diff --git a/examples/ios/requirements.txt b/examples/ios/requirements.txt
index 9d6b364ee36c..236ca6a487fa 100644
--- a/examples/ios/requirements.txt
+++ b/examples/ios/requirements.txt
@@ -1,2 +1 @@
-flwr~=1.4.0
-numpy~=1.21.1
+flwr>=1.0, <2.0
From ff4a475b4f18d32c73bc3d804b2e0d187fb482f7 Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Thu, 5 Oct 2023 09:59:46 +0200
Subject: [PATCH 21/60] Only run publishing job on main repo (#2444)
---
.github/workflows/android-release.yml | 1 +
.github/workflows/docs.yml | 2 +-
.github/workflows/e2e.yml | 3 +++
.github/workflows/flower-swift_sync.yml | 1 +
.github/workflows/release-nightly.yml | 1 +
.github/workflows/swift.yml | 2 +-
6 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/.github/workflows/android-release.yml b/.github/workflows/android-release.yml
index 35df8c8a9cfb..ba11e1ee85e7 100644
--- a/.github/workflows/android-release.yml
+++ b/.github/workflows/android-release.yml
@@ -15,6 +15,7 @@ jobs:
run:
working-directory: src/kotlin
name: Release build and publish
+ if: github.repository == 'adap/flower'
runs-on: ubuntu-latest
steps:
- name: Check out code
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 78b04c5138d4..aa267bd9d1ac 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -32,7 +32,7 @@ jobs:
- name: Build docs
run: ./dev/build-docs.sh
- name: Deploy docs
- if: github.ref == 'refs/heads/main'
+ if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower'
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 214c8579d450..f87db59773b6 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -19,6 +19,7 @@ jobs:
wheel:
runs-on: ubuntu-22.04
name: Build, test and upload wheel
+ if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v3
- name: Bootstrap
@@ -120,6 +121,7 @@ jobs:
- name: Install dependencies
run: python -m poetry install
- name: Install Flower wheel from artifact store
+ if: github.repository == 'adap/flower'
run: |
python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Download dataset
@@ -154,6 +156,7 @@ jobs:
run: |
python -m poetry install
- name: Install Flower wheel from artifact store
+ if: github.repository == 'adap/flower'
run: |
python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Cache Datasets
diff --git a/.github/workflows/flower-swift_sync.yml b/.github/workflows/flower-swift_sync.yml
index d3fce3b22a0f..836d905b2df2 100644
--- a/.github/workflows/flower-swift_sync.yml
+++ b/.github/workflows/flower-swift_sync.yml
@@ -12,6 +12,7 @@ concurrency:
jobs:
build:
runs-on: ubuntu-latest
+ if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v4
- name: Pushes src/swift to flower-swift repository
diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml
index 0ae9c43ddbf1..823ff1513790 100644
--- a/.github/workflows/release-nightly.yml
+++ b/.github/workflows/release-nightly.yml
@@ -11,6 +11,7 @@ jobs:
release_nightly:
runs-on: ubuntu-22.04
name: Nightly
+ if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v4
- name: Bootstrap
diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
index 207bb1283739..8eb9d88575f3 100644
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@@ -40,7 +40,7 @@ jobs:
deploy_docs:
needs: "build_docs"
- if: github.ref == 'refs/heads/main'
+ if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower'
runs-on: macos-latest
name: Deploy docs
steps:
From cc7db6ee340a070fb343eeb347166ec4236b4e2e Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Thu, 5 Oct 2023 10:57:38 +0200
Subject: [PATCH 22/60] Build wheel but skip upload on forks (#2468)
---
.github/workflows/e2e.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index f87db59773b6..6fa24006b601 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -19,7 +19,6 @@ jobs:
wheel:
runs-on: ubuntu-22.04
name: Build, test and upload wheel
- if: github.repository == 'adap/flower'
steps:
- uses: actions/checkout@v3
- name: Bootstrap
@@ -31,6 +30,7 @@ jobs:
- name: Test wheel
run: ./dev/test-wheel.sh
- name: Upload wheel
+ if: github.repository == 'adap/flower'
id: upload
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
From 5f504a91f4a9c0c93b0f8255e55c48f777ae8b41 Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Thu, 5 Oct 2023 13:24:56 +0200
Subject: [PATCH 23/60] Update the Opacus example requirements (#2469)
---
examples/opacus/pyproject.toml | 10 ++++------
examples/opacus/requirements.txt | 7 +++----
2 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/examples/opacus/pyproject.toml b/examples/opacus/pyproject.toml
index 8ee2cc7d10b8..af0eaf596fbf 100644
--- a/examples/opacus/pyproject.toml
+++ b/examples/opacus/pyproject.toml
@@ -9,9 +9,7 @@ description = "Differentially Private Federated Learning with Opacus and Flower"
authors = ["The Flower Authors "]
[tool.poetry.dependencies]
-python = "^3.8"
-flwr = "^1.0.0"
-# flwr = { path = "../../", develop = true } # Development
-opacus = "^1.4.0"
-torch = "^1.13.1"
-torchvision = "^0.14.0"
+python = ">=3.8,<3.11"
+flwr = ">=1.0,<2.0"
+opacus = "1.4.0"
+torchvision = "0.15.2"
diff --git a/examples/opacus/requirements.txt b/examples/opacus/requirements.txt
index e6e5dbb2fdfa..f17b78fbf311 100644
--- a/examples/opacus/requirements.txt
+++ b/examples/opacus/requirements.txt
@@ -1,4 +1,3 @@
-flwr~=1.4.0
-numpy~=1.21.1
-torch~=2.0.1
-torchvision~=0.15.2
+flwr>=1.0, <2.0
+opacus==1.4.0
+torchvision==0.15.2
From 990f59c5aeafbfce4e9606c73fb8d90505fe279b Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Thu, 5 Oct 2023 13:30:19 +0200
Subject: [PATCH 24/60] Update jax example requirements (#2466)
---
examples/quickstart-jax/pyproject.toml | 11 +++++------
examples/quickstart-jax/requirements.txt | 8 ++++----
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/examples/quickstart-jax/pyproject.toml b/examples/quickstart-jax/pyproject.toml
index 6a67cff6f4b5..41b4462d0a14 100644
--- a/examples/quickstart-jax/pyproject.toml
+++ b/examples/quickstart-jax/pyproject.toml
@@ -5,12 +5,11 @@ description = "JAX example training a linear regression model with federated lea
authors = ["The Flower Authors "]
[tool.poetry.dependencies]
-python = "^3.8"
-flwr = "^1.0.0"
-jax = "^0.4.0"
-jaxlib = "^0.4.0"
-scikit-learn = "^1.1.1"
-numpy = "^1.21.4"
+python = ">=3.8,<3.11"
+flwr = "1.0.0"
+jax = "0.4.17"
+jaxlib = "0.4.17"
+scikit-learn = "1.1.1"
[build-system]
requires = ["poetry-core>=1.4.0"]
diff --git a/examples/quickstart-jax/requirements.txt b/examples/quickstart-jax/requirements.txt
index bf7a9c64d66f..964f07a51b7d 100644
--- a/examples/quickstart-jax/requirements.txt
+++ b/examples/quickstart-jax/requirements.txt
@@ -1,4 +1,4 @@
-flwr~=1.4.0
-jax~=0.4.10
-numpy~=1.21.1
-scikit_learn~=1.2.2
+flwr>=1.0,<2.0
+jax==0.4.17
+jaxlib==0.4.17
+scikit-learn==1.1.1
From f6ed52b6e4e3cd5cbf15db8d256e230f930e76cc Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Fri, 6 Oct 2023 10:31:37 +0200
Subject: [PATCH 25/60] Update advanced-pytorch requirements (#2471)
---
examples/advanced-pytorch/requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/advanced-pytorch/requirements.txt b/examples/advanced-pytorch/requirements.txt
index 21c886d16e4d..ba7b284df90e 100644
--- a/examples/advanced-pytorch/requirements.txt
+++ b/examples/advanced-pytorch/requirements.txt
@@ -1,4 +1,4 @@
flwr>=1.0, <2.0
torch==1.13.1
torchvision==0.14.1
-
+validators==0.18.2
From de207b364940125c102a12a4b53ad4c20c646649 Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Fri, 6 Oct 2023 10:35:46 +0200
Subject: [PATCH 26/60] Update android example requirements (#2472)
---
examples/android/pyproject.toml | 1 -
examples/android/requirements.txt | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/examples/android/pyproject.toml b/examples/android/pyproject.toml
index 0ecaaa73989f..2b9cd8c978a7 100644
--- a/examples/android/pyproject.toml
+++ b/examples/android/pyproject.toml
@@ -10,7 +10,6 @@ authors = ["The Flower Authors "]
[tool.poetry.dependencies]
python = ">=3.8,<3.11"
-# flwr = { path = "../../", develop = true } # Development
flwr = ">=1.0,<2.0"
tensorflow-cpu = {version = ">=2.9.1,<2.11.1 || >2.11.1", markers = "platform_machine == \"x86_64\""}
tensorflow-macos = {version = ">=2.9.1,<2.11.1 || >2.11.1", markers = "sys_platform == \"darwin\" and platform_machine == \"arm64\""}
diff --git a/examples/android/requirements.txt b/examples/android/requirements.txt
index 6420aab25ec8..7a70c46a8128 100644
--- a/examples/android/requirements.txt
+++ b/examples/android/requirements.txt
@@ -1,3 +1,3 @@
flwr>=1.0, <2.0
-tensorflow-macos>=2.9.1, != 2.11.1 ; sys_platform == "darwin" and platform_machine == "arm64"
tensorflow-cpu>=2.9.1, != 2.11.1 ; platform_machine == "x86_64"
+tensorflow-macos>=2.9.1, != 2.11.1 ; sys_platform == "darwin" and platform_machine == "arm64"
From c236d14ed7a2f73d773032135b3bdc01ebc46415 Mon Sep 17 00:00:00 2001
From: Daniel Nata Nugraha
Date: Fri, 6 Oct 2023 10:40:04 +0200
Subject: [PATCH 27/60] Update advanced-tensorflow example requirements (#2470)
---
examples/advanced-tensorflow/requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/advanced-tensorflow/requirements.txt b/examples/advanced-tensorflow/requirements.txt
index 6420aab25ec8..7a70c46a8128 100644
--- a/examples/advanced-tensorflow/requirements.txt
+++ b/examples/advanced-tensorflow/requirements.txt
@@ -1,3 +1,3 @@
flwr>=1.0, <2.0
-tensorflow-macos>=2.9.1, != 2.11.1 ; sys_platform == "darwin" and platform_machine == "arm64"
tensorflow-cpu>=2.9.1, != 2.11.1 ; platform_machine == "x86_64"
+tensorflow-macos>=2.9.1, != 2.11.1 ; sys_platform == "darwin" and platform_machine == "arm64"
From 56b7d79fb9a5532c3af028887de408e7fa8e4a21 Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Fri, 6 Oct 2023 11:20:41 +0200
Subject: [PATCH 28/60] Skip publishing jobs on fork PRs (#2476)
---
.github/workflows/docs.yml | 2 +-
.github/workflows/e2e.yml | 6 +++---
.github/workflows/swift.yml | 2 +-
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index aa267bd9d1ac..da3a67fc155e 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -32,7 +32,7 @@ jobs:
- name: Build docs
run: ./dev/build-docs.sh
- name: Deploy docs
- if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower'
+ if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && ${{ !github.event.pull_request.head.repo.fork }}
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 6fa24006b601..3a58503ea66e 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -30,7 +30,7 @@ jobs:
- name: Test wheel
run: ./dev/test-wheel.sh
- name: Upload wheel
- if: github.repository == 'adap/flower'
+ if: github.repository == 'adap/flower' && ${{ !github.event.pull_request.head.repo.fork }}
id: upload
env:
AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
@@ -121,7 +121,7 @@ jobs:
- name: Install dependencies
run: python -m poetry install
- name: Install Flower wheel from artifact store
- if: github.repository == 'adap/flower'
+ if: github.repository == 'adap/flower' && ${{ !github.event.pull_request.head.repo.fork }}
run: |
python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Download dataset
@@ -156,7 +156,7 @@ jobs:
run: |
python -m poetry install
- name: Install Flower wheel from artifact store
- if: github.repository == 'adap/flower'
+ if: github.repository == 'adap/flower' && ${{ !github.event.pull_request.head.repo.fork }}
run: |
python -m pip install https://artifact.flower.dev/py/${{ github.head_ref }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Cache Datasets
diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
index 8eb9d88575f3..9edd7f7ff6e1 100644
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@@ -40,7 +40,7 @@ jobs:
deploy_docs:
needs: "build_docs"
- if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower'
+ if: github.ref == 'refs/heads/main' && github.repository == 'adap/flower' && ${{ !github.event.pull_request.head.repo.fork }}
runs-on: macos-latest
name: Deploy docs
steps:
From 7e14c96e1577a4a5fa9db5e56a2286bc569dc556 Mon Sep 17 00:00:00 2001
From: Charles Beauville
Date: Fri, 6 Oct 2023 12:01:41 +0200
Subject: [PATCH 29/60] Add script that generates the changelog for the latest
version (#2451)
---------
Co-authored-by: Taner Topal
---
dev/get-latest-changelog.sh | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
create mode 100755 dev/get-latest-changelog.sh
diff --git a/dev/get-latest-changelog.sh b/dev/get-latest-changelog.sh
new file mode 100755
index 000000000000..d7f4ca7db168
--- /dev/null
+++ b/dev/get-latest-changelog.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../
+
+# Extract the latest release notes from the changelog, which starts at the line containing
+# the latest version tag and ends one line before the previous version tag.
+tags=$(git tag --sort=-creatordate)
+new_version=$(echo "$tags" | sed -n '1p')
+old_version=$(echo "$tags" | sed -n '2p')
+
+awk -v start="$new_version" -v end="$old_version" '
+ $0 ~ start {flag=1; next}
+ $0 ~ end {flag=0}
+ flag && !printed && /^$/ {next} # skip the first blank line
+ flag && !printed {printed=1}
+ flag' doc/source/ref-changelog.md
From fa0fd8a465a489ece7ef2b585bbbc19b9857e1f1 Mon Sep 17 00:00:00 2001
From: Javier
Date: Fri, 6 Oct 2023 14:59:24 +0100
Subject: [PATCH 30/60] Update `embedded-devices` example (#2384)
---
doc/source/ref-changelog.md | 2 +
examples/embedded-devices/Dockerfile | 31 +--
examples/embedded-devices/README.md | 208 +++++++++++------
examples/embedded-devices/_static/diagram.png | Bin 358110 -> 84958 bytes
.../embedded-devices/_static/rpi_imager.png | Bin 0 -> 259665 bytes
.../embedded-devices/base_images/README.md | 11 -
.../base_images/cpu/Dockerfile | 43 ----
.../embedded-devices/base_images/cpu/build.sh | 9 -
.../base_images/gpu/Dockerfile | 10 -
.../embedded-devices/base_images/gpu/build.sh | 9 -
examples/embedded-devices/build_image.sh | 12 -
.../build_jetson_flower_client.sh | 42 ++++
examples/embedded-devices/client.py | 194 ----------------
examples/embedded-devices/client_pytorch.py | 216 ++++++++++++++++++
examples/embedded-devices/client_tf.py | 133 +++++++++++
examples/embedded-devices/requirements.txt | 4 -
.../embedded-devices/requirements_pytorch.txt | 4 +
examples/embedded-devices/requirements_tf.txt | 2 +
examples/embedded-devices/run_jetson.sh | 25 --
examples/embedded-devices/run_pi.sh | 25 --
examples/embedded-devices/server.py | 162 ++++---------
examples/embedded-devices/utils.py | 175 --------------
22 files changed, 587 insertions(+), 730 deletions(-)
create mode 100644 examples/embedded-devices/_static/rpi_imager.png
delete mode 100644 examples/embedded-devices/base_images/README.md
delete mode 100644 examples/embedded-devices/base_images/cpu/Dockerfile
delete mode 100755 examples/embedded-devices/base_images/cpu/build.sh
delete mode 100644 examples/embedded-devices/base_images/gpu/Dockerfile
delete mode 100755 examples/embedded-devices/base_images/gpu/build.sh
delete mode 100755 examples/embedded-devices/build_image.sh
create mode 100755 examples/embedded-devices/build_jetson_flower_client.sh
delete mode 100644 examples/embedded-devices/client.py
create mode 100644 examples/embedded-devices/client_pytorch.py
create mode 100644 examples/embedded-devices/client_tf.py
delete mode 100644 examples/embedded-devices/requirements.txt
create mode 100644 examples/embedded-devices/requirements_pytorch.txt
create mode 100644 examples/embedded-devices/requirements_tf.txt
delete mode 100755 examples/embedded-devices/run_jetson.sh
delete mode 100755 examples/embedded-devices/run_pi.sh
delete mode 100644 examples/embedded-devices/utils.py
diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md
index ced58029bd5a..86993542ea4d 100644
--- a/doc/source/ref-changelog.md
+++ b/doc/source/ref-changelog.md
@@ -16,6 +16,8 @@
- FedProx ([#2210](https://github.com/adap/flower/pull/2210), [#2286](https://github.com/adap/flower/pull/2286))
+- **Update Flower Examples** ([#2384](https://github.com/adap/flower/pull/2384))
+
- **General updates to baselines** ([#2301](https://github.com/adap/flower/pull/2301).[#2305](https://github.com/adap/flower/pull/2305), [#2307](https://github.com/adap/flower/pull/2307), [#2327](https://github.com/adap/flower/pull/2327))
- **General updates to the simulation engine** ([#2331](https://github.com/adap/flower/pull/2331))
diff --git a/examples/embedded-devices/Dockerfile b/examples/embedded-devices/Dockerfile
index add8d6d50d2e..ea63839bc9d6 100644
--- a/examples/embedded-devices/Dockerfile
+++ b/examples/embedded-devices/Dockerfile
@@ -1,28 +1,13 @@
-ARG BASE_IMAGE_TYPE=cpu
-# these images have been pushed to Dockerhub but you can find
-# each Dockerfile used in the `base_images` directory
-FROM jafermarq/jetsonfederated_$BASE_IMAGE_TYPE:latest
+ARG BASE_IMAGE
-RUN apt-get install wget -y
+# Pull the base image from NVIDIA
+FROM $BASE_IMAGE
-# Download and extract CIFAR-10
-# To keep things simple, we keep this as part of the docker image.
-# If the dataset is already in your system you can mount it instead.
-ENV DATA_DIR=/app/data/cifar-10
-RUN mkdir -p $DATA_DIR
-WORKDIR $DATA_DIR
-RUN wget https://www.cs.toronto.edu/\~kriz/cifar-10-python.tar.gz
-RUN tar -zxvf cifar-10-python.tar.gz
-
-WORKDIR /app
-# Scripts needed for Flower client
-ADD client.py /app
-ADD utils.py /app
-
-# update pip
+# Update pip
RUN pip3 install --upgrade pip
-# making sure the latest version of flower is installed
-RUN pip3 install flwr>=1.0.0
+# Install flower
+RUN pip3 install flwr>=1.0
+RUN pip3 install tqdm==4.65.0
-ENTRYPOINT ["python3","-u","./client.py"]
+WORKDIR /client
diff --git a/examples/embedded-devices/README.md b/examples/embedded-devices/README.md
index 16cc47bf3992..b485f663e08f 100644
--- a/examples/embedded-devices/README.md
+++ b/examples/embedded-devices/README.md
@@ -1,142 +1,216 @@
# Federated Learning on Embedded Devices with Flower
-This demo will show you how Flower makes it very easy to run Federated Learning workloads on edge devices. Here we'll be showing how to use NVIDIA Jetson devices and Raspberry Pi as Flower clients. This demo uses Flower with PyTorch. The source code used is mostly borrowed from the [example that Flower provides for CIFAR-10](https://github.com/adap/flower/tree/main/src/py/flwr_example/pytorch_cifar).
+This example will show you how Flower makes it very easy to run Federated Learning workloads on edge devices. Here we'll be showing how to use NVIDIA Jetson devices and Raspberry Pi as Flower clients. You can run this example using either PyTorch or Tensorflow. The FL workload (i.e. model, dataset and training loop) is mostly borrowed from the [quickstart-pytorch](https://github.com/adap/flower/tree/main/examples/simulation-pytorch) and [quickstart-tensorflow](https://github.com/adap/flower/tree/main/examples/quickstart-tensorflow) examples.
-## Getting things ready
+![Different was of running Flower FL on embedded devices](_static/diagram.png)
-This is a list of components that you'll need:
+## Getting things ready
-- For server: A machine running Linux/macOS.
-- For clients: either a Rapsberry Pi 3 B+ (RPi 4 would work too) or a Jetson Xavier-NX (or any other recent NVIDIA-Jetson device).
-- A 32GB uSD card and ideally UHS-1 or better. (not needed if you plan to use a Jetson TX2 instead)
-- Software to flash the images to a uSD card (e.g. [Etcher](https://www.balena.io/etcher/))
+> This example is designed for beginners that know a bit about Flower and/or ML but that are less familiar with embedded devices. If you already have a couple of devices up and running, clone this example and start the Flower clients after launching the Flower server.
-What follows is a step-by-step guide on how to setup your client/s and the server. In order to minimize the amount of setup and potential issues that might arise due to the hardware/software heterogenity between clients we'll be running the clients inside a Docker. We provide two docker images: one built for Jetson devices and make use of their GPU; and the other for CPU-only training suitable for Raspberry Pi (but would also work on Jetson devices). The following diagram illustrates the setup for this demo:
+This tutorial allows for a variety of settings (some shown in the diagrams above). As long as you have access to one embedded device, you can follow along. This is a list of components that you'll need:
-
+- For Flower server: A machine running Linux/macOS/Windows (e.g. your laptop). You can run the server on an embedded device too!
+- For Flower clients (one or more): Raspberry Pi 4 (or Zero 2), or an NVIDIA Jetson Xavier-NX (or Nano), or anything similar to these.
+- A uSD card with 32GB or more.
+- Software to flash the images to a uSD card:
+ - For Raspberry Pi we recommend the [Raspberry Pi Imager](https://www.raspberrypi.com/software/)
+ - For other devices [balenaEtcher](https://www.balena.io/etcher/) it's a great option.
-![alt text](_static/diagram.png)
+What follows is a step-by-step guide on how to setup your client/s and the server.
-## Clone this repo
+## Clone this example
-Start with cloning the Flower repo and checking out the example. We have prepared a single line which you can copy into your shell:
+Start with cloning this example on your laptop or desktop machine. Later you'll run the same command on your embedded devices. We have prepared a single line which you can copy and execute:
```bash
-$ git clone --depth=1 https://github.com/adap/flower.git && mv flower/examples/embedded-devices . && rm -rf flower && cd embedded-devices
+git clone --depth=1 https://github.com/adap/flower.git && mv flower/examples/embedded-devices . && rm -rf flower && cd embedded-devices
```
## Setting up the server
-The only requirement for the server is to have flower installed. You can do so by running `pip install flwr` inside your virtualenv or conda environment.
+The only requirement for the server is to have Flower installed alongside your ML framework of choice. Inside your Python environment run:
+
+```bash
+pip install -r requierments_pytorch.txt # to install Flower and PyTorch
+
+# or the below for TensorFlower
+# pip install -r requirements_tensorflow.txt
+```
+
+If you are working on this tutorial on your laptop or desktop, it can host the Flower server that will orchestrate the entire FL process. You could also use an embedded device (e.g. a Raspberry Pi) as the Flower server. In order to do that, please follow the setup steps below.
+
+## Setting up a Raspberry Pi
+
+> Wheter you use your RPi as a Flower server or a client, you need to follow these steps.
+
+![alt text](_static/rpi_imager.png)
+
+1. **Installing Ubuntu server on your Raspberry Pi** is easy with the [Raspberry Pi Imager](https://www.raspberrypi.com/software/). Before starting ensure you have a uSD card attached to your PC/Laptop and that it has sufficient space (ideally larger than 16GB). Then:
+
+ - Click on `CHOOSE OS` > `Other general-pupose OS` > `Ubuntu` > `Ubuntu Server 22.04.03 LTS (64-bit)`. Other versions of `Ubuntu Server` would likely work but try to use a `64-bit` one.
+ - Select the uSD you want to flash the OS onto. (This will be the uSD you insert in your Raspberry Pi)
+ - Click on the gear icon on the bottom right of the `Raspberry Pi Imager` window (the icon only appears after choosing your OS image). Here you can very conveniently set the username/password to access your device over ssh. You'll see I use as username `piubuntu` (you can choose something different) It's also the ideal place to select your WiFi network and add the password (this is of course not needed if you plan to connect the Raspberry Pi via ethernet). Click "save" when you are done.
+ - Finally, click on `WRITE` to start flashing Ubuntu onto the uSD card.
+
+2. **Connecting to your Rapsberry Pi**
+
+ After `ssh`-ing into your Raspberry Pi for the first time, make sure your OS is up-to-date.
+
+ - Run: `sudo apt update` to look for updates
+ - And then: `sudo apt upgrade -y` to apply updates (this might take a few minutes on the RPi Zero)
+ - Then reboot your RPi with `sudo reboot`. Then ssh into it again.
+
+3. **Preparations for your Flower experiments**
+
+ - Install `pip`. In the terminal type: `sudo apt install python3-pip -y`
+ - Now clone this directory. You just need to execute the `git clone` command shown at the top of this README.md on your device.
+ - Install Flower and your ML framework: We have prepared some convenient installation scripts that will install everything you need. You are free to install other versions of these ML frameworks to suit your needs.
+ - If you want your clients to use PyTorch: `pip3 install -r requirements_pytorch.txt`
+ - If you want your clients to use TensorFlow: `pip3 install -r requirements_tf.txt`
+
+ > While preparing this example I noticed that installing TensorFlow on the **Raspberry pi Zero** would fail due to lack of RAM (it only has 512MB). A workaround is to create a `swap` disk partition (non-existant by default) so the OS can offload some elements to disk. I followed the steps described [in this blogpost](https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-20-04) that I copy below. You can follow these steps if you often see your RPi Zero running out of memory:
+
+ ```bash
+ # Let's create a 1GB swap partition
+ sudo fallocate -l 1G /swapfile
+ sudo chmod 600 /swapfile
+ sudo mkswap /swapfile
+ # Enable swap
+ sudo swapon /swapfile # you should now be able to see the swap size on htop.
+ # make changes permanent after reboot
+ sudo cp /etc/fstab /etc/fstab.bak
+ echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab
+ ```
+
+ Please note using swap as if it was RAM comes with a large penalty in terms of data movement.
+
+4. Run your Flower experiments following the steps in the [Running FL with Flower](https://github.com/adap/flower/tree/main/examples/embedded-devices#running-fl-training-with-flower) section.
## Setting up a Jetson Xavier-NX
-> These steps have been validated for a Jetson Xavier-NX Dev Kit. An identical setup is needed for a Jetson Nano and Jetson TX2 once you get ssh access to them (i.e. jumping straight to point `4` below). For instructions on how to setup these devices please refer to the "getting started guides" for [Jetson Nano](https://developer.nvidia.com/embedded/learn/get-started-jetson-nano-devkit#intro) and [Jetson TX2](https://developer.nvidia.com/embedded/dlc/l4t-28-2-jetson-developer-kit-user-guide-ga).
+> These steps have been validated for a Jetson Xavier-NX Dev Kit. An identical setup is needed for a Jetson Nano once you get ssh access to it (i.e. jumping straight to point `4` below). For instructions on how to setup these devices please refer to the "getting started guides" for [Jetson Nano](https://developer.nvidia.com/embedded/learn/get-started-jetson-nano-devkit#intro).
+
+1. **Install JetPack 5.1.2 on your Jetson device**
-1. Download the Ubuntu 18.04 image from [NVIDIA-embedded](https://developer.nvidia.com/embedded/downloads), note that you'll need a NVIDIA developer account. This image comes with Docker pre-installed as well as PyTorch+Torchvision compiled with GPU support.
+ - Download the JetPack 5.1.2 image from [NVIDIA-embedded](https://developer.nvidia.com/embedded/jetpack-sdk-512), note that you might need an NVIDIA developer account. You can find the download link under the `SD Card Image Method` section on NVIDIA's site. This image comes with Docker pre-installed as well as PyTorch+Torchvision and TensorFlow compiled with GPU support.
-2. Extract the image (~14GB) and flash it onto the uSD card using Etcher (or equivalent).
+ - Extract the image (~18GB and named `sd-blob.img`) and flash it onto the uSD card using [balenaEtcher](https://www.balena.io/etcher/) (or equivalent).
-3. Follow [the instructions](https://developer.nvidia.com/embedded/learn/get-started-jetson-xavier-nx-devkit) to setup the device.
+2. **Follow [the instructions](https://developer.nvidia.com/embedded/learn/get-started-jetson-xavier-nx-devkit) to set up the device.** The first time you boot your Xavier-NX you should plug it into a display to complete the installation process. After that, a display is no longer needed for this example but you could still use it instead of connecting to your device over ssh.
-4. Installing Docker: Docker comes pre-installed with the Ubuntu image provided by NVIDIA. But for convinience we will create a new user group and add our user to it (with the idea of not having to use `sudo` for every command involving docker (e.g. `docker run`, `docker ps`, etc)). More details about what this entails can be found in the [Docker documentation](https://docs.docker.com/engine/install/linux-postinstall/). You can achieve this by doing:
+3. **Setup Docker**: Docker comes pre-installed with the Ubuntu image provided by NVIDIA. But for convenience, we will create a new user group and add our user to it (with the idea of not having to use `sudo` for every command involving docker (e.g. `docker run`, `docker ps`, etc)). More details about what this entails can be found in the [Docker documentation](https://docs.docker.com/engine/install/linux-postinstall/). You can achieve this by doing:
```bash
- $ sudo usermod -aG docker $USER
+ sudo usermod -aG docker $USER
# apply changes to current shell (or logout/reboot)
- $ newgrp docker
+ newgrp docker
```
-5. The minimal installation to run this example only requires an additional package, `git`, in order to clone this repo. Install `git` by:
+4. **Update OS and install utilities.** Then, install some useful utilities:
```bash
- $ sudo apt-get update && sudo apt-get install git -y
+ sudo apt update && sudo apt upgrade -y
+ # now reboot
+ sudo reboot
```
-6. (optional) additional packages:
+ Login again and (optional) install the following packages:
+
- - [jtop](https://github.com/rbonghi/jetson_stats), to monitor CPU/GPU utilization, power consumption and, many more.
+ - [jtop](https://github.com/rbonghi/jetson_stats), to monitor CPU/GPU utilization, power consumption and, many more. You can read more about it in [this blog post](https://jetsonhacks.com/2023/02/07/jtop-the-ultimate-tool-for-monitoring-nvidia-jetson-devices/).
```bash
# First we need to install pip3
- $ sudo apt-get install python3-pip -y
- # updated pip3
- $ sudo pip3 install -U pip
+ sudo apt install python3-pip -y
# finally, install jtop
- $ sudo -H pip3 install -U jetson-stats
+ sudo pip3 install -U jetson-stats
+ # now reboot (or run `sudo systemctl restart jtop.service` and login again)
+ sudo reboot
```
- - [TMUX](https://github.com/tmux/tmux/wiki), a terminal multiplexer.
+ Now you have installed `jtop`, just launch it by running the `jtop` command on your terminal. An interactive panel similar to the one shown on the right will show up. `jtop` allows you to monitor and control many features of your Jetson device. Read more in the [jtop documentation](https://rnext.it/jetson_stats/jtop/jtop.html)
+
+ - [TMUX](https://github.com/tmux/tmux/wiki), a terminal multiplexer. As its name suggests, it allows you to device a single terminal window into multiple panels. In this way, you could (for example) use one panel to show your terminal and another to show `jtop`. That's precisely what the visualization on the right shows.
```bash
# install tmux
- $ sudo apt-get install tmux -y
+ sudo apt install tmux -y
# add mouse support
- $ echo set -g mouse on > ~/.tmux.conf
+ echo set -g mouse on > ~/.tmux.conf
```
-7. Power modes: The Jetson devices can operate at different power modes, each making use of more or less CPU cores clocked at different freqencies. The right power mode might very much depend on the application and scenario. When power consumption is not a limiting factor, we could use the highest 15W mode using all 6 CPU cores. On the other hand, if the devices are battery-powered we might want to make use of a low power mode using 10W and 2 CPU cores. All the details regarding the different power modes of a Jetson Xavier-NX can be found [here](https://docs.nvidia.com/jetson/l4t/index.html#page/Tegra%2520Linux%2520Driver%2520Package%2520Development%2520Guide%2Fpower_management_jetson_xavier.html%23wwpID0E0NO0HA). For this demo we'll be setting the device to the high performance mode:
+5. **Power modes**. The Jetson devices can operate at different power modes, each making use of more or less CPU cores clocked at different frequencies. The right power mode might very much depend on the application and scenario. When power consumption is not a limiting factor, we could use the highest 15W mode using all 6 CPU cores. On the other hand, if the devices are battery-powered we might want to make use of a low-power mode using 10W and 2 CPU cores. All the details regarding the different power modes of a Jetson Xavier-NX can be found [here](https://docs.nvidia.com/jetson/l4t/index.html#page/Tegra%2520Linux%2520Driver%2520Package%2520Development%2520Guide%2Fpower_management_jetson_xavier.html%23wwpID0E0NO0HA). For this demo, we'll be setting the device to high-performance mode:
```bash
- $ sudo /usr/sbin/nvpmodel -m 2 # 15W with 6cpus @ 1.4GHz
+ sudo /usr/sbin/nvpmodel -m 2 # 15W with 6cpus @ 1.4GHz
```
-## Setting up a Raspberry Pi (3B+ or 4B)
-
-1. Install Ubuntu server 20.04 LTS 64-bit for Rapsberry Pi. You can do this by using one of the images provided [by Ubuntu](https://ubuntu.com/download/raspberry-pi) and then use Etcher. Alternativelly, astep-by-step installation guide, showing how to download and flash the image onto a uSD card and, go throught the first boot process, can be found [here](https://ubuntu.com/tutorials/how-to-install-ubuntu-on-your-raspberry-pi#1-overview). Please note that the first time you boot your RPi it will automatically update the system (which will lock `sudo` and prevent running the commands below for a few minutes)
+ Jetson Stats (that you launch via `jtop`) also allows you to see and set the power mode on your device. Navigate to the `CTRL` panel and click on one of the `NVM modes` available.
-2. Install docker (+ post-installation steps as in [Docker Docs](https://docs.docker.com/engine/install/linux-postinstall/)):
+6. **Build base client image**. Before running a Flower client, we need to install `Flower` and other ML dependencies (i.e. Pytorch or Tensorflow). Instead of installing this manually via `pip3 install ...`, let's use the pre-built Docker images provided by NVIDIA. In this way, we can be confident that the ML infrastructure is optimized for these devices. Build your Flower client image with:
```bash
- # make sure your OS is up-to-date
- $ sudo apt-get update
+ # On your Jetson's terminal run
+ ./build_jetson_flower_client.sh --pytorch # or --tensorflow
+ # Bear in mind this might take a few minutes since the base images need to be donwloaded (~7GB) and decompressed.
+ # To the above script pass the additional flag `--no-cache` to re-build the image.
+ ```
- # get the installation script
- $ curl -fsSL https://get.docker.com -o get-docker.sh
+ Once your script is finished, verify your `flower_client` Docker image is present. If you type `docker images` you'll see something like the following:
- # install docker
- $ sudo sh get-docker.sh
+ ```bash
+ REPOSITORY TAG IMAGE ID CREATED SIZE
+ flower_client latest 87e935a8ee37 18 seconds ago 12.6GB
+ ```
- # add your user to the docker group
- $ sudo usermod -aG docker $USER
+7. **Access your client image**. Before launching the Flower client, we need to run the image we just created. To keep things simpler, let's run the image in interactive mode (`-it`), mount the entire repository you cloned inside the `/client` directory of your container (`` -v `pwd`:/client ``), and use the NVIDIA runtime so we can access the GPU `--runtime nvidia`:
- # apply changes to current shell (or logout/reboot)
- $ newgrp docker
+ ```bash
+ # first ensure you are in the `embedded-devices` directory. If you are not, use the `cd` command to navigate to it
+
+ # run the client container (this won't launch your Flower client, it will just "take you inside docker". The client can be run following the steps in the next section of the readme)
+ docker run -it --rm --runtime nvidia -v `pwd`:/client flower_client
+ # this will take you to a shell that looks something like this:
+ root@6e6ce826b8bb:/client#
```
-. (optional) additional packages: you could install `TMUX` (see point `6` above) and `htop` as a replacement for `jtop` (which is only available for Jetson devices). Htop can be installed via: `sudo apt-get install htop -y`.
+8. **Run your FL experiments with Flower**. Follow the steps in the section below.
-## Running FL training with Flower
+## Running Embedded FL with Flower
-For this demo we'll be using [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html), a popular dataset for image classification comprised of 10 classes (e.g. car, bird, airplane) and a total of 60K `32x32` RGB images. The training set contains 50K images. The server will automatically download the dataset should it not be found in `./data`. To keep the client side simple, the datasets will be downloaded when building the docker image. This will happen as the first stage in both `run_pi.sh` and `run_jetson.sh`.
+For this demo, we'll be using [CIFAR-10](https://www.cs.toronto.edu/~kriz/cifar.html), a popular dataset for image classification comprised of 10 classes (e.g. car, bird, airplane) and a total of 60K `32x32` RGB images. The training set contains 50K images. The server will automatically download the dataset should it not be found in `./data`. The clients do the same. The dataset is by default split into 50 partitions (each to be assigned to a different client). This can be controlled with the `NUM_CLIENTS` global variable in the client scripts. In this example, each device will play the role of a specific user (specified via `--cid` -- we'll show this later) and therefore only do local training with that portion of the data. For CIFAR-10, clients will be training a MobileNet-v2/3 model.
-> If you'd like to make use of your own dataset you could [mount it](https://docs.docker.com/storage/volumes/) to the client docker container when calling `docker run`. We leave this an other more advanced topics for a future example.
+You can run this example using MNIST and a smaller CNN model by passing flag `--mnist`. This is useful if you are using devices with a very limited amount of memory (e.g. RaspberryPi Zero) or if you want the training taking place on the embedded devices to be much faster (specially if these are CPU-only). The partitioning of the dataset is done in the same way.
-### Server
+### Start your Flower Server
-Launch the server and define the model you'd like to train. The current code (see `utils.py`) provides two models for CIFAR-10: a small CNN (more suitable for Raspberry Pi) and, a ResNet18, which will run well on the gpu. Each model can be specified using the `--model` flag with options `Net` or `ResNet18`. Launch a FL training setup with one client and doing three rounds as:
+On the machine of your choice, launch the server:
```bash
-# launch your server. It will be waiting until one client connects
-$ python server.py --server_address --rounds 3 --min_num_clients 1 --min_sample_size 1 --model ResNet18
+# Launch your server.
+# Will wait for at least 2 clients to be connected, then will train for 3 FL rounds
+# The command below will sample all clients connected (since sample_fraction=1.0)
+python server.py --rounds 3 --min_num_clients 2 --sample_fraction 1.0 # append `--mnist` if you want to use that dataset/model setting
```
-### Clients
+> If you are on macOS with Apple Silicon (i.e. M1, M2 chips), you might encounter a `grpcio`-related issue when launching your server. If you are in a conda environment you can solve this easily by doing: `pip uninstall grpcio` and then `conda install grpcio`.
-Asuming you have cloned this repo onto the device/s, then execute the appropiate script to run the docker image, connect with the server and proceed with the training. Note that you can use both a Jetson and a RPi simultaneously, just make sure you modify the script above when launching the server so it waits until 2 clients are online.
+### Start the Flower Clients
-#### For Jetson
+It's time to launch your clients! Ensure you have followed the setup stages outline above for the devices at your disposal.
-```bash
-$ ./run_jetson.sh --server_address= --cid=0 --model=ResNet18
-```
-
-#### For Raspberry Pi
+The first time you run this, the dataset will be downloaded. From the commands below, replace `` with either `pytorch` or `tf` to run the corresponding client Python file. In a FL setting, each client has its unique dataset. In this example you can simulate this by manually assigning an ID to a client (`cid`) which should be an integer `[0, NUM_CLIENTS-1]`, where `NUM_CLIENTS` is the total number of partitions or clients that could participate at any point. This is defined at the top of the client files and defaults to `50`. You can change this value to make each partition larger or smaller.
-Depending on the model of RapsberryPi you have, running the smaller `Net` model might be the only option due to the higher RAM budget needed for ResNet18. It should be fine for a RaspberryPi 4 with 4GB of RAM to run a RestNet18 (with an appropiate batch size) but bear in mind that each batch might take several second to complete. The following would run the smaller `Net` model:
+Launch your Flower clients as follows. Remember that if you are using a Jetson device, you need first to run your Docker container (see tha last steps for the Jetson setup). If you are using Raspberry Pi Zero devices, it is normal if starting the clients take a few seconds.
```bash
-# note that pulling the base image, extracting the content might take a while (specially on a RPi 3) the first time you run this.
-$ ./run_pi.sh --server_address= --cid=0 --model=Net
+# Run the default example (CIFAR-10)
+python3 client_.py --cid= --server_address=
+
+# Use MNIST (and a smaller model) if your devices require a more lightweight workload
+python3 client_.py --cid= --server_address= --mnist
```
+
+Repeat the above for as many devices as you have. Pass a different `CLIENT_ID` to each device. You can naturally run this example using different types of devices (e.g. RPi, RPi Zero, Jetson) at the same time as long as they are training the same model. If you want to start more clients than the number of embedded devices you currently have access to, you can launch clients in your laptop: simply open a new terminal and run one of the `python3 client_.py ...` commands above.
diff --git a/examples/embedded-devices/_static/diagram.png b/examples/embedded-devices/_static/diagram.png
index 66d8855c859f33581bbb428823ef0e3c3fe3e340..7eaa85fb24c66d6fd307adc371fb28110e972bdd 100644
GIT binary patch
literal 84958
zcmdSAbzfB9_dcv|1yqm}5RgWY9J)gqBqXGy1R1)!29)l`0i!YsFqCR7F|#m$!EZg1}#y+bxur!Rk7#3p5e>=aK<&&q2Wex_&TmsB1i)^*HW+q?RV&8!uh
zz4`}-udc7BW@fd(j`hv$%DU!8w%(@B0rPq2N>!fOnMu|r9PdRWL}XRJhb8ceN-NfR
zuFQ{n1Vn|zB!>k1>sfjp9-XW&O-hv7&He7@<>ad>%Z7M+XbyyaGPIVG`7k@t?d9RL
zw|~fyXz}N8S5wDeX=Z3*a!Ov;OtIe6*FPv5R-)bDw~Ig||A;Vd4qe~a`q=S(c5Z&C
zukp)hY)ovFVvCpe_lO52
zQAMpQJhy3kY-@Wva3ovvTVZ@kb~>!|*HC%OR99AWKDh9E-`-60c#%eqdt6zHe}3eb
zSm&zhYOyS^vxBW;c5uLaiA}uk;^|J+WOK(r-AAo2t_fwxt-9g4X_Isly8x%1USmO3
z8P#A53pa`NCMm~eH(>=aP=nXOn2nLXLD$kCtBe?Qqj6%$`p=Z#&{XFQaxJ3G8P;X5
z&=nb#Q=|2nH$E%6vabys@ku$bzNOcOOq}!TXncLxkYf}hTHpwBrTE$>66zHlFKv>21pb`~>l>p+&a>}?GIGZS
zLr7)`;3v3rMy8SL0S4@6$S}pWcmI~b&+$%>?kS6mxwo>;|F;wyhed*1;tWem+qXmK
zzt0SiZ%4-otaZ2w{#DoJn|K+WugMZIsML2<)|45KufMU7TU@}P69KjJ+Pl6xI
zcu&Rg>76BUD6-`mlk>uNa|&rRg+wOHZ~-yu%!`<7+a+Jgli8TQGJNxlgJgn~G$6dr~UElQ&pL>5T_GT{veuBx<=OseXW
z$z!NT{5f2g3J_khs-0AQO$6*f(X9;2`|~~cT^#z=l1V!l{)xJJlGRB2Bg~KyP}c<)
zI3a{5E&%x6)19nkFbDdj#TUO<4?i$-KzqQEbJ{nduZ9Y@kObXXE1i(Lg>
z{Qx~?XU$>Mz7kadwr12bgpGoCQm2&5D(4aJu)?XH;Ro{aWQV_?bPu6n3|RTd^mu8@
z^7J{F7O5kR`!w{4*Tj8dWw^-r-9w7GsmK;%UkN#r2?h*TVVTtH{J
z+^G>^HhA=L+A7|{|IV=S|JAZ%_#0&h~@+h0MdZ{A#Q0Q
z32Cr4V7NG0VH79RZdDl7vuuCzi0~ZBEW8$W7K-ft)g(%rkS>w|v#G_-_eR=J#e?{o
z4+^Yv7NS**Xr4nfV@*yz|E7+5_kYXAtYO*ZWQ2PJgb~OC%h3g1or6F9PX#d6`w2RS
zf4`w)e|v!|`${T9L;N+-Ut?jZXYUS4iAeOs(!C0jMwc;0alXJ7E&7YM{_+5*$OgI6
z-%}=6_waP%q5VS|M@SYZ#}ClgDFCCVNbsj_9<0A#nD7&*8_Gyu=+yAe0rtW-p#s*5
z8!P-jfCZ6xvzLlh(Ejc)BMdZ(4S47FdVSYQXad@fhFGl%Q+N#s1DfS{EomqtA~9h`
zDF2@_iZhFA=kl$yIG{;@AwXV1_z}CV85#i40OTbl&3Wl79gcB+z_7q6-Ls
zA0(ZjVe*Lor9%LITTjBLmnE!NzqzGwO=jB{cAvlx9FaOblCMHBWg*G;Rv5T&a_@h3
zi7}FpKymKjg^3pRQzdQjcTcQrTh0@IFgGB0|8ITJ^SvSSD3uAbsbJvnH2jDmWRqrf
z!)V&Gx4h)Cc!5U1K7bB!O?K6LCzBt`I(yJqB>V#pzQ53$ApX)NxAh2SXB&0hHs&>M
za)>70QxX`~`~N#G32w7Q?gUhTc+agS&D_dKdVZAQU>#61pcV!6?0*{Xe{TkII~%G*7lE=z-KxoxZau-$S8hJ<~`iLnwdDVk^4%RsFrRg
zb<0498P8-J2f$N*IS3W5Nt{umQ`0O-p9p!JJ?e@GkYKd7ktZgUSLWccIH?8Cry?enXM
zKw~}K$a1tCp?3}`}R7iIyBnGytOoiQ#4l(W8B6&C!?SQSRL&iLSXY6?;ajvpJJEsOK-~Hz=P1R6>_A74uJXUF2!4SfQ&oXVT8Jj{cp1Hy6nO!JD`F
z0HH8%QGf<)!0Kim#6PWTty6K_
z$oa7TUi9ZOBbv259-xT*ALC$ioP$vqOxcG>?|+;rgyk>|&9Xt3`XzIj1#QiH9r~0o
zF0|um{H373)IfI$>`-OL{-6H*UwGpMDI0E6O+gWk6FQd^aj=`qL5@4QO4-f7yd9JE
zSg4;!R^S*FM*#<*W;8s5v$%OB*^DWpmqobpEpN)lk4vU<~X1JZT*6j2A5^=h1RPx3QV
zF)ZFGFAs9TbhG}&LgAgAgN31t0iNF`I}j_CovO(V%YE)j@0v5`C|@+7G9#5~Pz{?=
zEkggiXfrvRWc>?ZBiy=W?F?S*PoO(K0U$wA!`t9TxLd@jAYY<6Qp8
z_+>|uo?$@|QW0ecsDK;Dc9hF0g9W!^cW{`5+U{%^zDTiEk0t&-2Z8{3R5OfZT^om&
z)p;k=vo9#4_8y$qDp2nu9zP{ibdwf1ty%17rKUa7L4~RqJDJflo9GfyG_4p
zYa7n22)S>DS=`-?UK0
ztWI;nB5CC(DA1!Qt(B^PTA%1|D)dyml2jH#!fS7BymH-gG4BIqt^`>y6a+8(SSWc^4PCMyCTj0$_#5Z
zVO(SJ407}RO_+Zfe1PQ5Ik=d&QD;0sb~8mUwsO<5`KRnVR53PzdtiSlsES!^1&x3f>O-iX6u`%yv`Du!U`ELLAFu-F)tA)DAQ_A(tMIK3+NJ7!6j7!0P$^?K%KZvPu<^rWw)rRG}1F+F`V&CqOAzO&qK
z%o6rsDhD4&43)cnWE~mIinMhdi$v)Q4;jBFfLQU9NULN@
zhHq$3#2H#73kUES5#Q3ATi
z;B^s@upVZrwRxAO5HTPmoQmFHe+X3qDj__lE;)8c%
z+Qkh`A2*zC@_LrVrW5}`HrIb9k0scXD+t{MV8%bFE4Y@iw-=ko>XNQT1m9m>`eGoh
zpsznfgd^W0lFpi^dt_Cxyi|G9N`&KqX-J4w8!=o|)Eg2-rRo5OW;N;;M;a7OjMql@
z=V3AY3vDle(6)K@#in*55&6as;?kOqHlS}v5aX?>`kI$Rrjn{vh`+Kr>M3GrtI6%S
z6(`Q_BRfdD1+3t59ttPee25QeZORk2Gr2zc!HN{QTHkAYukh&aG5zguG?*A>#HVH4
zCj2$3sB+(p=KkGNSjs7bcLcZ3QxYSqHgmT5s?psOJJ5wzBv@!9-TP;y&{xO{`VFi%
z-kJ{2X?eF;U63VaCJ0;73IC2&2=10!e8{qd=$}oEnZwdi{}5%N{id}Js_orB@WGxYmWspKP6R_sd5sZ-`7+Gxu#Q*bF5Rh`lZ(EaeSFl?g
zAdYU1{&85qu<+xT7@Bwx^TavWoJ{{~qEQdhxn?pAP%Sw>lyMTvfgM6(*-S4`aR3go
z3Nvej5R%IcoTdc)vV6UZ;{2)$dq@{}e=V0{5Y57H{DTZ9anen;)Bj*1_Q2?@u^2#IFGUO3cvHI!M3
zHjrX=%fxu*H!y?!9f|6ZR-rY<`D
zFfIW7o92MTNv{8iTr|mjW_dBQ?e;y{GCn4~DzTPgKTfgrSuwW46nlF-QV#Zz2_*4m
z#`{CRr|pZd9K+xezbq4t_`nKFcuc5b6hK@%Mh|aYP@FB+-@1y+oVs{EvlMEOiJzuz
zpKS`LL0Hv>y@sEK@75&W`vXaBYH}HgQst<|bQ?GJ2)i@uxh9FV6x+)%E+A$7RhkyB
z2v7w(B-z+JXXfa<3e~vU4JR`(YSlhftV^!!aLPq}FnLYg5;n7h$#us7R8p@A^bKLq
z5faR$vprIbq6lZk(af4gQ#O?8(C%E7
zM``_UiL3QrAjz>Qwl4RPLN6kIU2v{~gUspQ28ty%frcS0AN`2NE3*|Fn&m<)-|0R6
zCXs+l+&Yg{tOmt-Q4~uYS*R`b-~xNd1r%lIJ~x^F=F@>5E8A$TOfHRXd;|Mr?oD9}3|f0ADITC^>$c!BaW>WAr?kEjj{p4zb&&V)H&Axrz9wJIr`nIF_
zOc7F!{Cw^N?tnk#y#;w+@ve1Yexy6@p_WtbSlE5Mm0;YcUkxvqx+*W`^dn%nn8*Gk
zNCOS3WIz>f^8-kC=oe_C&kD{9JuTH=V&F5qkN~>hJbY2d>p&Twvjj7zCibWCY
z_BLhu4(Rg;#4oSxF-zqfl672X+^f!p#xz*fMPi{80uEhE!h@XePVn!izl(*R#lne{
z{L;0iwdV2dDDQf{9w{9Agq|L~y{sTUJEdoRQ)RCpmPa%L{`J!^?tw){=h8BBP`9TD`AnPnB=3PBwljZ1Zpr
zQ;6Mr=EJ=-_}}0?@z6mdx`###1ND`Du3iBNAItSOgE4FSW8!RLmeyO`agBOCHlw%%il1^~l@r~$c
z%6tK}9N%7BN$${!pZciOjlfh|*9iH*gd-CxUeVjTkxO(5(HSCk%LjNq2Y<@V8I>9ELlj|XJ!
zh7S6441<;wNui&Qm*;Pd`zO7qJl#Tlc3TQ->{8Cb&nPauSp2^ENWva|2@DT9b*ROo
z>eoG%iH}%PZWaW%b1@;vXfiUg1?p^QLISrUXhIg*j4B_eDJaCjsH0nhzK_Ae4{nXn
z(Ol6TPh6SGo4Oj;2IqP)XrJno2(~X3w%v{>J_O>K0kG91)0gW#D`-{Zh(3!mYwVrp
z8$ofh%?Mn0jlUv4^h56%9KSw;obQbF2|Q7EuYQ(vgFgfp<2757m$F-|O6x|>G03a9
zh3taQ*PaL~fX;%tmg_G!ap@EsI1LES!BQDDqzrld15EyhFY*PlheV5vC?>+W?ucbl
zLrX!D9H5%=1i5wt1Rm^
zd`V^$tDw9tAzhUOtJ;May`Rx|^q6t)#%-os5vi-0PLr0FH9MHC%C9&P$M4`1lgE{8yupacl@#`^-57~K+m4M9l`(`6YtDXC
z1*-&HuJe+`dHw0A7rY?>BZfAezSq}ti}rWynen)vx@R})GtW!hRkSe?Y09;&4ZW*M
zTlyQ5$wu01D%9Gb}Ds@E#
z5hlAz)LYv$MvC!M21FC8ru4c%KD5rhAmWMzA*p(hykuO%R;#=S1d;sQevA#%=|_T@
zxb9@hv8yFIARZg(JVLI`C%t7yq7b6&%e%{0NTjXAaG)cEFtYiut1E$$oILaMZq!
zH`nPr`_qTtp57t%acg#NUD=ktGB=(*&N{eX9Lnz(6SJ|lz9o2$c=>W<c(Huj+sjL?2=G*9L-o#Mi@35#q2uLBZKuc%s>)99e7du91Zi8Q?>BoxKAK(2R
z`q~>I;PJzzvP3|@=T;s!HI#J1Xpfw1SFRKe6HvF=DV&2HAWBA7F#G5UkK=?VFAGU2
zL-$?vy(pG1CKT_wGcAwrWah|eJvF(cxw;)xeglb$%TWK!0JX>xiWBbw$O;unGs&!u
z4X50D(#eJ7_{Hu}9270SwMV`c?xHsHi?qn!>#S`5Zx$d{OF7EO+x`bzC656|bZO
zyTM7xjwN^Bd!l1rmR4E;=}!=nCq}TAAVxd2F{Cj@Uj6s*RZ8T2BDRn8rZDpSjepMZLt!s%U6R7P(tU`z}tZ_I=0Qod}s*u~qK*g;~jq
zn`pPi>SzCXs!r(rYNEXP644_R{43zi%iS`{C;QGAgG`;`rqlnGTAz2YV++y+u(o0q>xp(Ex<@PSr3x|8zc
z0Cfv5Ajkna{yA7}tmv{h9i8WrhwSp;`%`k7YlX4bUG}~@m*+XsPWd9u1BXZaJTmKB
z>SAB;(zcLxl&-gUXSD2uTD8<~u%rE`1+WluNA2Jm<^-<6(go>wdqGX>cXqwwSN)XV
zT8FPI>C+2>N3o7VBA)@dRepI*avsoxUk}l78ROgLZlarV`;LVe68`To1jx-{lF7t|
z4l80tovcEAk-BC?i_r1MZlYLjH)F05Ug#Gy%ibOlY4K(&v+BmvyGw2&c;~VnUFQHN
z8B7t&923#Gg_MG5GqRPq8kEARFCLRp`bci%%)?G%DWSk+4_Xt^L0zZlz{Cx8Vbg_l
zY8wNEu&LC;D*>?G2r`3Yf>WBWTRafg7!KYYN+PoU?`CpQW=D$*;F}*^c9L`kr^8S2
ztA&&^eC~?*SMRZATzqlwAzM%0rdPQ(l8j|B`BD##krHAvWo>vqs%
z#6cjk)PmOCVe+O(aN%yv_6U;F<=1JqKDqEKyX=miwwX-q_HpyR)%{K)8=mq1#
za}t-&YE{2ZseXE1yZ4?yg4R)vtv%`!1B4{8g?DSjJ139P!n+)=AqmCFy5HwGjcNFr
zMqbbwb_t!Jxou~sLFxUz?Kd{H=;NH}Q+3TplA<`pE4ULfe1snVhRTZ&cO&Q5VJ^oD
z=!-(i1!!(rY4
z8CT^VRX#NE#c5(PvFASV>^e#;m
zIl`tCKGrFP{{s1+1{ADHUj8i)oCeZi4_h!2Y43LZWo@T*S+&N83sBD8rj5Ain#6xM
zkBggV=ak=|I2(tb4Hq;?u_|wHDe2V})$?IPlKr=F1RP_gmwQ}MFSK^`9!Rtd7p^OP
zv>sZeg!Xo1o?8$AdSSdh_3F5wJmz0qI}8IU%~U)(2L}<+Wlny%$AjEdh~$c0L0u+y
zzcnp1d}niFT84Wi^T|yu*E#I}sI&q5-3$4dkO2K*j!&YynGW@uWz1i3fg5-|VbRK=
zWo5Ut;56hJHU8Ys`9b{1U24%i<&Qem219D@Qd$?J=5mx??CID1CMyAT5V-(_H&Bju+y4napJ%bz~^;ss=b$$i;ME%RlL*f6_uy0E21Lb%Gkk8lwZ^02GEZCB!*IwG?`cT#b{mz##+etL8SGPsNeHVM^6z4$>tf4+5Ar1Sd%}qxNCa&X#2oRcG4r!#X)WmZqH}$F&C_alXgS&{yHQY5JiU9
zjv=#a%Set-%PV(DQH_RGs}7`zP|^D5f{bnl+U(2A#2yV_
zoM$&u04Q4i=zeg4#7#)p{pmgW2y~}_SWj)FieFaq%Kpl0e&^qkYF)>f(r&%@KdV$D?o}!vqXMFM-JR0#Q%c9;tN%KX-nI*v&7gu@>JwXe&
zM+z&TNUG;R?aBMD@0aByKE{_s+lDXX9Ag@B=7{+e?=5o8)y~=m!DVs6p5;ilc3}}-
zy;J`@B|r~7%|y((WVDVab(538O*H7Y|DVqaXz}KN1ZqrJXNPEf@xPj#4?k3|$%*R%stT7d4mqtMH@Z7M1hjz0+vm8dX;
zA7=YIRFgCs#OLZVA1f&`8Dw5$LZn6?4NrarME;SpEnC|)Bt@4i^z+amdDXp0hU4&Q
zoT8L=f>eB4!XE|F39++e0dkpr??b~fUCS+msnF8;Nc?1eo3(AFQrRzJ(%tG++m3FF
zX#=0r0S3wt9Z#(}d@LDzP3!@5^j{zXV(-m
zp)|_n*Z0%J%hJ-@^cKHQ%^`v`Zmle(9rg4+MVzDnaYUc0267HvBvV6@IpIe7pp&Tn
zo@xR)-h;fAu$j-69_tNb+IepCf`*>4TU&ev&B@BcBCV@3SC{G^KmW95==`$MBP{0S)WnhkS(r)M(3(Dw$pY#p$UCA({Rt*1Bx)tVVO$3|>=L
zN{>J3WnG6B#<&nhV7AWxwV_p!d2h&s9K6Ev=^B;@1C=`6mXYSKW9b+ccmCpN
zDHMIvO)nTn&d+c7wirZku4j{_YmJ#;AgJ$A##%ZucDyCCFM`JP^WbAnN@Y(~RI9k$
zPr!x4lbnO;(+P+$IYd_Z&qoQ(h@>cE73x~CGCI4hSYPVlemE|6@!6Op=Bs$#5N^9s
z5e$0aa6K!8r_=$NuXdo5%zwslS9eOJ*B5y5wyH0>?J{_NhI2(%Rdc)M)e0WnWO+xu
z$?lN(mNd}pC65iaV2H7nf{<`uEAWij7*j(mf@_j|Xv4QRSAH(}u@{pUt%@H0a3HWve%}xn
z;0okq-<(0?dT`-Zpgp&rZ*DN)4Vo>V`uc=?faq#{{87Zrdb^<3WtXvFNm7_@r}tJ%
zQe+q5*rv?>wHJeOF>q7H2(zwu{WkY6xv(Z^&2_nh`?J!HST=lxe=zFKpMz(E}-Q
zXqQRBg=U`sXae5Jq
zcQES-z_k`s4C%v@jO{Wp_Ap<(5Plr&+PFcl%hHn!R1b^a=zS6E(iKP_T4ML$m%JN>
z8yt8Rop(QI;!;;QznkpQRfmHwv$NA=XJ^{3PY>ce=+gNPYa)50SLz&yMyzB@bFP|gkFH%hBd6(bL
z+NFL2>9p&A@ymO;q
z6Fz~}51^Mgamcz&_B}P44bJ0k{$;+%`sgrc=sBw1VdE%kWx}0@ZhJ%CPqZa@bC0h6
ztCVadd0&XA3f^q}=+wFMaAZj{vH!_uo5vY07@o;gEF~Q{?oy?3cgBCoPUxMXDb5bH
zp-~dWf4{|%alKr5IKyyI?RsWn$
zfq|s)^|+fd8J{hiT@D}b(VQ}kr{wc%->Q#4621UZP*O~2*jYZS^Ji|7ojCN1LwL5H
z8VnVG8qsk(efF&Lh$nT?n%0q2otGS#dVX;ZuJmJcl}>(79=={3JBllbS_f;U7?9+p
zm1?C9ztS@{(HbF+LgG3jkhPQin
z=#`nDX%e3*p_#M&01gb~pF)`oR{P1CuH|{h5;lYO0x~6>t@Xi>rgjqYy!LOf?#gE;
zUaR?GFsK!E8^m$mI)ESKQnoalgW9f<5V~8NBG~lSGdcpy^fWYC=EKdw38>25;1pHy
z^iCI-ZpnU!M}sRWtOAEAw@+AWZcwP(DlWl4xlqH*%&%N{_@gyEs$beB-wBdT0(i2A
z=^WoQQ{hPW;&BPfLBJma%5q5T>q)3U-J}c5`hVu4PvZ2Y`}5Hlk>*hdAK=e9J>rFE
z@`*^okyB9|`YiS86pJ0kC{3cVkCcwGb;`0a!(ZzS0=Z?u$h4KD*NyrvXXH{MK$95f
z)l$=o%uxdcv!!b$Pc3(t$gr-xdhNSDUV!*9w=N=UV)EbTHrABGcQEHR1gLB)do1%Y
z!?-MFphoIR00}C?^aSAVnOew7&T6a*T7~Z15dSSJKdj>^A(oixqrGNXokC=ghT_6@
zVndV)-EC;%@JI`#5_vx&sio`4P=`|6p`jvpqEd2egi&+v$S1234I~Brwpk|w_3+H@
zE3@nXBv@iT05H(kFq&&prgB>Dab}?wgDez9277Ulv}UTj7g-u&h($KOt(v32JGY&0
z`PGNxMRk`m29}R2w*%5=9Gy>u&N;|m9LZXz9L2;ve3J(HzAtc$5#$5L5l__3+-nS3
z&3mH@K$35xd9u^>4AEipS;eiyo;xCR0>I2oxO`HuTdutO6G8o*gfMv&CtxFv`2#X5
zYA1}P&cQL=yJ}p}kL8kr%Xgwh@DRO26nEXk^*`o~tW2Z@55tAJZGeE?=p3(mMJc?^
zyMVZW3$fOVWp+EI#I=djpsL7E{X=!~Y2`>cND(>C7S$DJLNTt51)wUjPtkP+A4hqY
z*9xr6by|3bBWD6MkS#_RITlGcgP$oyKs!=W09;c6)<2&cB@cTy9
zkh1YUBp_4o($4m0ZF}eDNrmTnhNf(Qe_>>;+MeVrY`LN$O?{4B4KW@_MT?H!jw`P)
zmK2M3K;jp^V;G9TdOcWLXm!GLT+4o^YeosRao{1ru46M#onat#Ea^)F1I=TmObu6?
zt?OY+uW$W)Rv)zjbw|iPWy+w-_6C&d@
zJNQKdLs6UvBRGqaa4`Iau>z?qqS0f}_%p1hviy#qAni!SKKi>+S`?K|P(>T>V7^${
z8&{vN_%^Gr2lEvTVSQVJ#Em=p<2-iovTSCf*6C|mV16M_T`9l>O5lfr?s~f+X#=_*B4cU!R(XmxsF~POY5Q)NZabsjC(x|A~m|l9jZeOCJ
zxiYj~Yf6}=+!=1P>>(C)Cjnd?^N4YN8{wK_7^|DP+HLpJf4iTGsk->$HhWbuRkWyd
z19LhWoc9U_9{+McqN
zcvbO7+LN1TBWR%!{Rs4^op{RHg(>;vMm^vf&4~WxT5&O3x6KHX;L9_#?cgX
zU=(L^$5C?OXx$8kw`C^ik6X$*7foJfdVA?#<-%h9>Fm*2sQ(Z#p=${Jn
z=##w%v~x%dTn=mzk
zO!c8*5n)ZLOa^swWCNjvUR79~RhH=StlHTuW%1e*=lki-_i25!gQF}_V`e;eAt+9l
zkSzarJBzha@B4Gpv`UO02k%
zMn4ga`EcCqB5HQ{dAm>Mo-n{DBhZ=^;!(+7qe!?eSGH3@Azfk6|=@A+?_vAAyrYxS9tSI{T#W
zxx`R?+KNQv&j2F@6C2l6S}F$%yU-KQ_S!
zwe!3B&bN`Z6C+6Bl@DVb39Q8r)Z2qikv{+W@beWOFin}^fM%M^gjr1Cz`#y(y_&KZ
z-WSKQ3)hV-$S}u%wAHUcG6^0%cbJWx&Pe7bJeg;rWW6%
znH;-Ne9TpgdWajdY}s%j?J?ugFJ~NPdef=M1LoEq^GgzykNsXF>-^If(iiO+|yA26N
zXTf{D%-;JRLYcUaj@RElva~34mM7ch1OD3>^3_8rWVh0#aYAsV`dSc1S$B<4iaJC_
znH|?+Ael?+7r3o-XwD~uTA_$GOip1{vfadGQk|a)^SF+s7Dv${G;ZAuk;|VnYdXiF
z1$lV-+qmXp}#SFyEYHBZCeNBq`{+U4DM-Y#I0cCOp1VshNACLOa7y2E|&Q|c|g
z$rCvGy3!RAW%9^Hw!XY`jDVpam(g8cV1^r~mlGUrYA&Dg+cJkZDacBE(*C8lMw1`!
z7?NTrA##n51RzUjh&MSIW%^4JWD%u!r`}C%<8=|Qm
z{R?!qulV5h`c!J+vpQ{K3DWji*Jy)#Y|u#_F=({HTXO`xQO$m)W)fQbxyJJOz;>IX
zkc>0k-K=ivt24!tSTw%Czm6FJ9^5V-8#F?3#yH03HfJh08Y9pQr;N9gDY%6vsYQqt
z=N=snHVq}*CVfw#wCLPUwk*!|@OD+=f5DPpK;xL^Juwop{Jk^GlA>p%;_^E{7bzhs
zIGt9k=iP>$`Tf&~VRaXTi*Fu*qt%1wlG+AGJVsjf^^(`>Ax`ezE&Qh)Fp8O*p)-%(
zSWc}6lD7@ZbCazq`k_A|vza|tJ$?*BaaKnzqXE;+N;Rq4mY1WsK*;^0$T#U^om>
zf;(XdT(3MOF`ab{c%GQHESIbHA*}mA);3>gnmZ?=UjSYA%&UjjGg{lwOqoSrX%
zf0Z$2`2mOl)m)5wY>kQOV#sm6U7jtCnigCh~@ug7D#ISZ5+>w~-R8WiD>?vol%G*Weyva(@_7^Fq)~PA?{Fq&i>b$x_OTbhP7ZDQNs3+2#ioIa5mRu}W
z%AvQvrC>!TQkHdm5V$MG+t@l>0u(aF)-RoBxiJHstL$C;sugqLyVY8CaJM}s%sc@%
zMm`#^N5_UBW}iA7_Fscofd
zh6d4uU6V~OMbRquV@O|0lU-w?EE1by44RqD6sOD`S61stzl7%~#nb{I7pUwUswZY7
z$naz-Ji{^a?<-&dQI>c*IUb|QhA)rVdHx}2}PC4iDNE5QJylzciJvS-C-
zXcA3=cUQa6>RLxQ#&=Zj*Nt@H&OSp&jTj0iBHA8t7MCACn^e~lvCeb!V%YTvK}5fv
zB)3WlW7K^GAwi#n4wRMfsM#%xtT2<7o|}Jd)Sd%^UWCy~F!x4%n|gzdiH=L=GC=`u
zn$9C_V&e`uJvSRITGww?p|t}uM>6@)t&dZ>M!W1#bLZI
z)VN`CfpfBtmQH!vXk?3d=QaGMz^4~$&PFvE&YKKHj4@?&0)B|4+6w2$j(#KHda!bT
z5rwSR^W>BI0xi`S3Qk<7G~6SiK>_X$*&wuX9e)Z48)?^46D#GM^F0d9jV|q7Tv-gC
zhSqFgLngHj;;h0sUqUX71$enj)<}CGp~MwZ%!~3IhZ?zDBy(X>k?`G(|HSI5NrtUQ
zPyo~lyfE>GM@%g!$E@)Gqv|Uks_3G<1r(%9x?5VhLmH%{8xaoO-61I<0s_*_p}V`J
zySw8YQsB__j?caK!}}lXnb~`-wSP6(ZOrx4oY+X^+DjIA#2?`*7xa;reBDYPkEaJ}
zfpS?J
z9BtS=a@b_=UqPE3&Tx05$PR{Y
zJ=4Q9XlJ*Ga7D?(D1^99~j^ciQMNd7r5EaLe#-h2@q9((L;3Ng
zo5R%f1k1jR3fk8}V_W(?{&D4^=p7e5p(K8XhH~Kch3?)7thzdB>WyCtyj;IeGastT
z#AP{*^xazTeZ*t-4t+bMVa-JqmyL_YN{FEDZXKx=6vIPjj^?2v-swt*-hPY
z->z^_pvYV$1|z#T#mvlh?Xdql0HsOdf6^yaICCL4rX23-p%KHUxwjLd{FoI?Vk1Yd
zN(`}9s#_iqW@0|!8meNl%>-Y^7uF5!uPTTC?qDv-ZylLThAT-)kIrG2oN;j}h#{_c
z)7oNJ@-|Y9N-=MbgB@L^f^FK}BU3{Sa6T3f%Q6+z2>H3k
zXKO=@!k^Z;u0%_7{~>swpC9iUo0227SU+i%Q**4bw4XH{9tp{Pv$ri1NrrB`Bhmr}
zpwQ)WcP%_t4U!DXHzOFjyc`Dx-X0ZJ4WzNl5@1z7qR1R}b&@b=q1UJF{tfl!UWFd=
zVK#`$%x8)hfHo5Vc{!&}Bst5}ybZ-IBOjJL)7;j&d0Ph5XNdoL3xSM-;2Y8lwGI)!
z{%Xuz+xcArp0P+&G1;z9^1@+hADM^JC$ATD9Gn@Yn*Lh%c&yv$mhA6e&WS6!i_FI?
zw;#dyb51^@!g|;+q4pHiOP@%lhGf4JOWM)C<&;RVuQ+b@T~Wxd|U08s~|D
z$z)vss~VDN#kA_;qws6!zB<%m4hnDJuWa(!Y$ja_^TU(tW%~_y%e9YWOG`|BS2h%Z0Gtk{P3R2PQou`H>b|!*!A0b
ziy+3q)@Q=QJt{`>(wo#>-TWyG^R4ajN>mL`Ve%E~G&>qeYD3XdTNFU&S%csM74ep8
zOkPLk&hETf_u&ROocop)uWmm)W4kOdx|vP}P8MKXkMRaKsx`{J=IXtGA{*IqbAToL
zdKTnYpiXyX{lIi%GHE9KUNE0I6FTasq?)d~`GN4J=ikD{V$E`}-_7T;)qIRUDZTVY
zXdrJstVQv$5;Y!dK?wah<^T_ohSZAr)L){?M@=v-Bsaq;J=eU>gW10y?R)?4``PnJ
zU5sP_mo=hw-R;5?m>WxWrQf`sT_+BJ4wL|}L+EH*i)O3bZbD~ddWsCfyRWSfH%2|f
zp`I#)QY*cCPQ*WRJsreK{V5hLb_I611^l?l9cb46v`8R|X=F`t#KR-F<<`$AhjVJ7
znaYE`LT+&W>`b~%T|C7~hZ%C=|JxVJf|A#FfaHQB*TE?9AmkS46^vPWzO1C(Gi*j5GI<#}48}luj-IR)MM`OqE
zLJ|DQ9>A5~0=Nw39M>GSzQEv5&)`xelGw2{xsMxoH6F2GQ8V@A;`1fHHAuO6S9L}Q
z%bsW{fQsP&@96E}29AeRa`}w5-_SPB_X8P(r)$jPjW7Vl3VkosQ=%!8(I)rW+rTvm
z`NAW1U`L%e+wE(_&Ehl#V61YgWtKPYw+53yo1s6IMP^XFLtv|^xh{a!?f=TV_$Xpa
zh$|Vg$uh3$Hpll1Dhhq&1p3`2@F2sI-l2t@K9dC%y>)7ZTlDXztAe1-{U4vlq4A{kgR2g133p6
zP=KjiViL?h_$Vsfm<(h{tU&&kf*SZTbt>5(BC|}qgi|kmt;<8E*1TC|_#JUr!rhLR
z9yXpidwqN%6DG$5cun#;MsW!;g`~VLD~!$W>-lm22pUG2WIS&%`yM_mt^kF=Z+xS^
zhUMS6+pXf8WIhu1cfMFUVZCJ}LYeK2jC-0$PBY(#`vkT%eO1TN~
zP2~~jl-V=Hok?Iu6MBS!*wV?U93L24d+zd&Y707#Ei)>@^}zvWfPmxgN#n-z&qrpv
zG65%0!(+?fr1(*ap17M)LGPp>9<&);_;=3z%7akw!u#=`o#QVX{X`6sr?{uDL4%(jw01xvzqqsDW2iE7<~k2+se?Q+M{UB)vkP
zf857{^pZ~N?P}RotwzO35jc0Edpb9wCi=urizIsma%ukz^O)>@DGcix+XB1Lc=g+!
z>*MbbgmZ^D>iZnaH#fR_d$n5AHy@N_<0UjEX5k+w<2^4J?FnU~R-0%71w3M5@>3R?
z*1G+`z4(jNne}!i^alnMmDEe-5e+d8HG3xS3?$PM#?BzUsBY^z2H{kh!34aC?
z+1yMeQCuo3p6+R{+7y?$Jz%E{Yl=dHcRs{(Jm4d}WiS<)JUoZ6I_mC6TDT(L;X9#x
zVb|~&Kgk%icUQ0bWNm~0oZpqvzncQeW`ILEsN1|(-k83}2no;y;KW!QieuToZkJH4
zs2-sSY>8etLho|c%MhlTNS^sM)Ewd3`$f2QLqJ`n<6lir5h7K5W?NPMdUN)Cd5IHn
zOBMxnNIijp74YL5wZ#&G65-ly6jZWxM%eYmx6@=ScdWIuf8!Ui7L4i=BL`y
zVTdhcCT=
zy(~N7WAUO2R4Z*r-_~Vo^V5Q82|%*^st4z@Z{B5S??7rXB%R$;?>u7IRS&`qU3~O2
z(i5#;46$N%Jm7=>>Kbllf5_`|l{D_J`oaYmdAH&zG%eeB9pfj(wWsO=nuNct1@jyGO^I&*RaEzvev)H^Oy}GF^dNwITSiWC^q#{s
z4;X=$My^2ofGFO#h@DFnC&4#{WGxiAyH%Czm6|rAQnwRp%Apn)Ho+WdlX{l^N||=z
z-yq3Gyu7d+g;^8$L?QF`Wr9NN2{v>%Gwf5WlMRy1nV669)ax`zQ6{TH+T5mf`
z+`KP-Ctd`Kc8Zd&zk=R1Aas{#ynLV?`?aE&$O^$OyMmSwp-BFr}jpi<;G_HZT)TwiZwJ
znGQLRHG!^fA*`jHa}{Kjzgy<`nFis$3i*9VvjzIGVd2Y2K}7F$C4&Y=$I$SWxRYz)*qV9}&nFo|5&
zH58YvD%UveogW6;3uFkEryz@2cBQ@XYn>>m{a{;{x_^E}n}4>zmOAj`BxOHMLE7K=c0NPyCqHvHEuNC>+jLGcP7}|&;Fr96MCDa
znWc(JoVu`#>uMTf(s%X{6Y*Syy%S&|4a0V*Yy5=mXChC0Gt+rl{?fZ4^cZlaCYts*
z{V*7JsZL>XfYggbo=Co=7I#4rnIEo`Sbn=mJe
zr_Fl@m&T~yCR+L+A1~tmwBIMuA`M8%=6ajt2F-5g^e%+P
zy{^$XqtWrGO9B9{E8o<-JXu;EG?@1p`>+xzc6g&41^
zNu>%fquY@yu9zMwd&UaX9KYrMJW0Z1x-@dIc*PyhWJ?r);-_`@ZXatl+3TpZ@;rto
zNVAh~VCCgDZNv8fSdGk-U*C^-6#|}yMq^Ewzt4?(1jJSB$@?M%9jcmsH{SRh)p-*s
znu&mX-za6`&q)kXF?ePCyHTv|Z^%yaPulhg!f@u&x8X-ly!LY0E7%ytPHE{801iVk
z{ZKdiDVz${<>hR4TPE=~nK+ZS%ZB)ph2uuxig5>~YbWrQ%I~=Q^$PS(!&qiRrt*u}
zWt>U&U7QBuKCri}{UPEt%I=LiUu*St+?g9?7jC>|q;)zyTR~mUMeQcFHe2mud
z=)`_q&(sFfCax#!2b~9Bs{Jipfa{X-w>N>F~M&ix&AtT540UY)ZT{U+=82@?=mo;ugU^!
z8`ZFykt0ENGVZT$yUg5!Y|0OGB-p(j%?5fyH)v<+Zz7)YRmD>cncY@O07hVHLS5y=dI&bdVm6qyV6A7sO?D0X19k
zuL5`KapeNc5+Q3mt9|@%Pg?`9YuK8Dlyegm76s#%g?u0%L~z*JJEv|asc3?!)h>f0
zzoNc|{oO}#3Xc)i)Cke9eVs2`mjREJ6Ka6d3g7rGQtT3PZPJH=vCcmU3^w_ZSbe^oZp0VTq
zupSn6zB9l+yUzN{t9Q%0AgBXf|4={b@ek(QTUj0z1B92uQTyZz*4`Y
zwHu~be}A^EVen2n`qoeDrqZgXe4W^9%_ZpIK{GleeR^X_6PRT#JXXSaqz{LfI{b~KKtUK$a^W_7|DWDWbO&iwi`{7KDNTWBGs^{`wmKUq1Fn?M%Br_d&UI{<<)BZLO#Tl~74uH6~vVVO4
z0!bJ-n_>a%W`G@VY;0#BcB&jngX}E@mLLR#t@D{d_o0ZkjgU5@Zx!1dw{P=@xiRSS
zby+wi0w02~(f|)biJmK8=+@#8a)s@-oy6?x_sUsw_`KAOl+W1Bw=hsx$-eBgygsJk
zZuI^Ib0Dcc>B@&XGvN-b-K*Ri&38$C{17P$A^W5E$s{#SGHOey^T~q{rg#W
zofi$#EF&z`Q-)BH=D;_6@pB?#2~B_rfjGy`le4)MSwJnjg4j7jxkS){Z*)?rHBGLk
z>hm%b6_EO`0#Q6d^-};0j4r8jN%LluSVxhZR&Vb0l7*&fa`DVPl9wi{U_)hpg(^r8tG_h4Wji0qU&GjW
z+{L87-UEuxTCk+zg0FE3pMis)1~$eG`g1)cRQgKMO{x?RDa-OoE95vR^{#GlTq3^l
zR_>abu(fK<^;-1P0MFOVDf->*D_xPy$%yp44zz^FO<)JAUDD9e&RMhC!vjxvSUbF3
zozJ#&qgApUhLyP2N5*uSNqm9X{8GnWbN*@Z*1H%>>{XT~Z>WG745WiVh65YuaG%jk
zjp*3@C6b8%P6BR5jw5xs(l*3$?=67ODij4-p?J{qdDv|Wt=r5r`;#J;HgqR0Zy?1+
zgburSX|64TWYXCK=1Sky#l=Ns2`@?cp|&t+op#Pw(5@y@Pd96D0Ee8Oyrry8!lgJ}
zX5pD&O8GZskV=duGiw(9`*GA@8P$M=S-4zj*A;i~U#aHQkzBwm%Ji?2((e_hKJpMp
zJ~9W>&XgNfZg=V#n&9$<{5-qzMwllnDqxdTzf#z{%UNXmK9WLvTswRPg8at-GLUkw
z{2)GmgedVDx+T)*j=b@R3U!c>%KBh(RhK)^K!V-AWsNh{=#h}_r{ePvy@%R3ZnOUY
zm5+jSZs9|`ex?ubV1nbnKt+H(yYd4cPRKZy$`3g!=D{
zBB{>=2hJ+CmU(z-j3#eHA90>0|Jtah5)S?mGGzb~(YQp~Yp~g3wFH&zKAFJo-j!`S
zl3UcGxbg87$j3?mRf97;DCU^}CL=s4I*culPZ#*Swc@Bwe^0-CfCu^S=Tn+19t^`y!#c^Yk
zOWjaS-rMRc;21^$&7S9-#n(UodSWAnz*y`oPDn7#`3Nf`MCmOAuo6?xsj5TXA&LMA
ziy@eHN58M&R-PC_e6v(s?Grraa4-HbWX>zQm9_#g2UwE<9jAq*+}MW3j*a|&z}obI
zlW5rJcW*{K1t!6MIt(K^^YC!s;n}s6+7MiP<2C~?a`s^QGWMB@Epx*kl>X+1w(s&{
zk2aIN#oBZy)9J>Ry85wFMfAAA(-A2ICc>TFm?hZe*f2xA66VPNwM{JG(*_*-aGQl#
zq=mJC4yu(o`WpYG?cs61W!nJ3-*4`%BHQX&8tu=est;uZh{qX*9NRa#AR6at#U;`e
z$ba}e8rfK^hPoa?5-&6%CShrNR=Ki{wJ@kfF*I3zAZgG`sqz
zhwe)^e}HRQ4Wpv*sj*GDIl|LZnx}B1v@7^qtE%@4l$&@)`;5gKkRZXzS#tSQ);rv6
zdUsn}Tlt<{09GwoHhd$Y*YM@$&qu#bChEou1(5N-&0Fq)k4Kg<)nqa8gX{xwVC)>KB4gdd|O)VlI_Nxv^x@R0vWc>_cl$pC)GufLL#j8L1!hufx1
zCy6vFJK$5pqRyLUiWsKOd8M(dl`UL0an10C-0au?s$F@;BGLY^+4Y$M+HiP8hawd~
zVa9oi-q;*fa5nGVHKCm1-;94LxS5-Y^zR^}zPB|=D>t#Z!IO4r{S%?=0;}WC=#yb;
zuRW=g2VE8n#Q#-CApiuNJDFrh#5d`pPlXSHZy^iKbb|ik)c5liLgSesIX7N(Gl2a;
z+lRe6#(jzcJ**dL|6tFnJe_Pv1{Pm={uvrdWxU3ewlKI{@J)Eaa25g2-S1eml#+d?
z{L+(IUxZu*OntWXy&;1QxmSjxHL{#P_?~kpR0=k|>G#)2Z|=(*ZqZT}&on&B&xVNj
z^mC|M8MHYI&Fp7l%6@;_^NY&yGS2J>`XTj>z-_2C++$(-HNd5TMH=2Ke?9dQ_tH151K&b8VPST-g$DW
z^T#JQA>N85$?ZF+ar^1>cBk;&(36K~Z6E&3hG#?XiKrh=(hQg{nLE})M;WQ4r#mw8
zi)U}c7s9#q+TUhPde+gCRuR`;mgZlNi~#6h+X_+;S)C}VtI;~0{*h_+T0HOj;Gd(~
zWgLJBeS4RGO)?RKGhLXIa~FrN@X3(2u+lR4M%^~z_AloW
z#c12wuAcWpjWhm>G(eaDA`P@)D+2bRe8jC5Gi$K@u>JGejQw@KE^oo;_F9n!S&lLb
zni2rEn06Al!?KyM=?af#$
zH=ICAsp}O4-O=^WulI`B*UmqQikXM}3Jxh|8O7lBIIqZ?;IN_tV}^wjlcwmdwxx|3
zqTTy<{@q<%^o;jem&G@bNo=%-&Q~KK=Xk416#)lcwXo;K)S-mR57dI%>5R=KMUayC
zFt3})crN6`byif^nUHQa<~#e0j%i
z(^MhAkNQWVZNP#~N#^;qzPLXjyHc5Yi6Q1)$@ZmgjP>xo<;=HxPK3O=y3jrtdn~y~
z^7<~rK_;iuyDYe^)agH_N6zo&=cz|6G>y!HF=?(D{x13#8P94)oYW4(Kpj6@=>ulD
zG?#U1Wns6dr@umP*o1kupKm-Bk#cUzXBbAX+r8FuR}@a|JB^|ts0UWzoiH7I$WU3c
z%kKbGl_9S{mHvLc6?9tLjfOm@om3H$oAl?oN+XNzhp=X8L=WOpTd)y1m-*<3!*s-Y
zlLZjH!|*Bhw9tX$@WfSWoCIwrOvOE}Efrig4yPN*F-BfI8!XhYmY~AwURFX54*A3u`hf7Hd#Uu%qI~Ad2PpeFN*LH
zY-%9>Y5U%Dw4eWRdYO1=y}ev?uUVK1AcPEd6Eny$n^kx^RmtJ8|E%-$R5e$y&zQ@1
zJXgA1zs`__Vd9`bZrBXusP7}NF)JIEp^B0fAvS}CO7M^BLJGtymE-zN^(Jc5(z}T0
z?ClesHNNwSLu)!d$E`ls+!erJzF@pH^$h(i(z>9r
zID&EFPs*DiktEmLJKD3WVQ|Fz{JhVP%vGJ$@Y8|;#1!H7Q
zUMFIQg1*;c&Pj?b>_0;sEsD}~=cv}}4ut`4B(5}pw;JJ^GJfLg75RspwB$%_t}K1{
zWmMdyDzu1RokT-pW|<+L-aFRVa~6rlwz4iRZ)->6eGOd`=rb=P{QVx4U`O)8#q+?YsB
zkjHll_2Jr8huLR(ZO^Ch5Kq`@&*Cs>(DLwn9{dxt3}#W=Ig7q%0S^2e^SJOC5FsA3
zU{3}Q=nnVO+G#&ojG_G8&sh;lnez5fOQ)i0(rjyDDtC^T!Q{yGLW}pl4CA6~_$?7i
z<;eBrg-`8(7R{#h%4`}*poIN0iSbS^pL>!q00>gXtmu4R4?V|UrIIx68Gw@C5|n26
z|CtXPa+d0Bd1{qDYxE$YcCP)DeYl>zr1CNb;MNKoaHy`LTmDTn#RyJJ>e0XbL26oU
zXWq9-8CcUd$4j0-!i)~WQ-RRgRWSW)T$LF*Ub4UVamT;jHfHpzljbn({~o5
zVz1b26%~HiRgcN;z@ii4gYp&Pdpz~rMm}Z+eoi7ly?A3=DqFc$p|SSSsDEKuK2#i1
zTa*%yI|3AeIlz}Vovi!oA(aBkKrIEEaw&eTS-1>!?{(Snw&t<_1b`emzco7W_t*F2
zahgN{#|lOB3vA)R*3^?FGRo8y5lFV4&NcgCXfwhxv`|@!?%LyVm;E3_n`m~-A|9%LX!?j-$us={S4*w?f`hf1krsz$hFV?ArP?0yOP4-B-?!3)dM&(RCXO|H75jh{H=ET6E(r3kZ#iXZw(Ay4wnqQuWM<}!+%*y(O^gf@SG
zrG2CI?|mq0r`MqQvGatrI^lK6kbzSe9PVFzVCrKdo@;l@B%0zWsiMt&iR&;O>utYq
za2PlV%yh!&R06F~7`0z!(w)w9dO!W--}FCF@r06vZ5s&`_qog~^X&*oy=dn>RWaf8
zGT%Pr^(cBG4!uM6EX3FioXw1tq1ECG!pqW(kDifzorT}|o?SB8i>&O+zj05KD9&X%
z+3TZXnzQyL&D!s$p#^(Fzcuja?`O@Hp(H93ojH{*w9!yvSr`hGn0J2O+P9yqD4z{M
z`u*Pb;?(%7Hgdofxh4Nsf-Yb)EWLHU4DtT6AHowmJFV)4pZ=S7<)_kdPq6%?|q0a_Z`-4wxBq{7TYDM(1kMy-ZLz4|SjNfXw6
z4Z|{DJG|dMG8b=UY2{Ac9mX0Z}Nug}2MDJ)&t=25MN|
zZs&_-T-|6sA)Usu8G3!Pi_&EVr3NM4_c`%p-#pKBy#P7WWeUZ9YDe^5ieCPb4i^ao
zzHw&yfe`F|kLlI@L?H?W;3JyT#$~A3#`yLf3~HNE-k#rTwnoj78yvPeSF?@w9TMCi
zK0!zzsIf-aqon<`%DcJlG|Z9NOTWShT>;`$1U}F=#yo;61|l+vrhg&{C}H*Dw6NvQ
z?6%Ln-OjB*<8eRtMmSV20mfLhoi}MV7C?35k=eDryv#B_tBX0sod)cyk2+q3qHph6
z3+Ccf=Y1GSy&*Z_qI4m!W|-XIuZfbcD=gghKV5gp`yzYBSDQ@QFHbGj1he%}w)$z=Pto$p+yF0QYK
za<@k@!6a})Xb^Q3%zk(g1Yd|hbdB9fv>~ePd9QHKFkfn=umdq+@tRyNZJ5h3(GurV1xCD#A8rv4|zjOF`xuf)^YS(77NfOYzUE9F5
z4#64rZt|hchK@a=-In2}68NqcaHg9`RS-fpa@urG*fz9mE`N9K$Xunt9S(HUQH+Ji0gGq2EtvVv5Wy>h2Tx>b2#z
zp`lSjO&bN3Wy8NSrC_}cab&zyiJU0O;m^Z_^owExT3z_DSXI>Tn_sm$nmX#Zn_{Kz
zr<0=F-&;^-zr;MA&9$|$ZqGhI>{4L5^$W4l{PwqH!@SVlR4s2ge?ewyqx(8eO#O_Y
zx3Cb~M)YG*?OQRV1g3fzMnF-sKwK`@&_c#IiH!o_AiJhmT7hnd2z@`O+kd{PUWU3R
z#K^$HO68?CmJK=mE;6sL_ek*VGx*ZZz(>HJ8Ek?wjsE{+8U<+K#yADjlBmEBPtm6U
zt-%d)w*ZGv8+ld_O4#3BkDQUAykiz+wPk``!x#Okt!}r_!z0%a!-^e}3(j2NMeEPmbO;w5*ih_ip1QXM>(ya>x@xjnnt6~8K?3eI<62?$
zfC|>8fExk~09iv)i9B(w-eiMKn%2z>Oera@INCqjS!C1B
zTe-=+OZn42bE{K%0T~EzhJO{^e)>KweYz&}y-w~FemvBL=Gctuow({bzF-4y;PmWe
zi@8VJPkBYzcw_~N`z68d!Bt)Ahcv9TQS@4r%;$jE+4b6wybVU07QRDz>S&xReRC61
z^6)Q%r)kN$*x2Q6h=H%wm_3E>Or^EXObREvz=5byBD63I+`Q8V=*TaGQ=A
zsN1}9=fboj1;;j5R<*RFpU_hf%>_LPZX5!_Rk?m6ui|=bp^vHII-WjB-xI9W*_M2K
z^KZCtzWRD-{iVQ$(ccpR7<$yef{6TW$nNslsEs8*)WCbAgJk=T^u~qZpm}2X>%Cz$
z-xBR`+xz=WK~CB=lWhH)D+j0P03M&j_zJY|r$5UU!hdlqjBdn)!*2#-+kG^|@rm88
zJp>Yomy+=AIUnkX?nMa`j+osN&F5NXG)$;O&ft-uFEEejh(+ZI!7NP&P{G7Ot+Cgx
zJq{a}UQ~_wBGIh8pt%uP(~X4FZ-Wps)YY{7Eit4~Ceb&M
z-*+Jb8=S<_{PTuB_JGgGWlYTA^3TNDGh9BDHgm&A;nEOs0qBCp_koF9n1@-4*~H+mA&myod4p9N=6K7H_ruxKX=B->`c*S8_RA8J%GdZ_rd)~
zBoGog;mXdnr~RRm%fc*M#}%zrT-J66WF)6bR_R|Bn>z0%cPGV_D9vJFliNuP}M4KuTLg
zO$Z!9iihL-o+v6B3>*&Mwu{T}Z~ecN@b^z4OGi*LvS8*$$YDtuVkM7xppCo%>%>kS
ze#7~>&qd$g_hLCqo0`x`Z#A87mC6eqgHBkf_^hAZtvz^M`(Fb=ZeoLplw7^9xIP@)
zY}~kQEF{8dFJe5_F17WaVGh3Gms#+uaoAv(d5_`j?EPV{Y`Ai)asL5c~1dC;$P*iXbh2LGK4|xU!{E2(UDQM}UUUt=y02Lw#HadGpeLzp!MjB4yn=#DB;&w!%7HJA+3eaN^K#Q&lP#}z
z1s^!IvzS;PN%4%O0GTJ@OYLFqxRm7IJH7o|Ih}itl@0qirFDNL+;x!L6Z_&ksbi#L
z&q7S9F$}G^(Ep|Lv(a%&1otNY%<=VLeK57A9OSR+phI)FDP;@Xubpg49G$tY9@*m;
zrim>ldY-!r*x$!|(odqTi~COFC9fFkaEZ|}DGqFN=Uj&D_lY{iC<(q0`HlDXD#nT0
z=@GModsAJNvz@h4zK@N*O9^qwPuH9`L}XPg2}V5CN@d<0YK*>vT(bC@FFkDQms($L$l3W7E954SKU`5hRazh8}``MLO#m^kB9?$^;K
z?4hb#C`$$B_op81C{e{|N;v0jS`sIZ!DgnKcU5U!OX@pf*
z0-Xk|_3&M^YR1;9+tO4`$Y8%M)c$zV8$I$$?voSx-_WIWpwOI&)s$lb^nso+
z*LRD68NpKJskJ-QAY%HL8DvoIkT*pITZ$evXi{N(q0A((U>
zcXRcYn3jk@@VYBjKQFT%3Qv<$(MkXl=$P>?>oj>f?Bj{HJk#%@}z6Tc6p
z9?Wt1VyTu-?>!56ZLOb#x
z<%p7^)Mz^CUB?MVyV*roDW{Sk2(${?v{9RE9wzXjDVN8DRSuh)kqW-sDc&pHi9Vg5
zFMn!Lf0gI@^x$0CaXHMmMub%J^K)Bqm&Zz|^F7WI;kC}(R*^wL`0BPPa|{CwO-7;&
z|40l`yYMXU(*1|L0nJy+4ieJ>#S=q5j{2eBW-8J4_cN39&YFvL!~?9V$eaaiTaptA
z_P^bLM}A;ZDQ`Ue$&n0qZ?$P>=$cJpJooL1D@nt(ROoD#X6}5mU$(!;P*9}CT9A&v
zUxchfG3idM9>R6a+4PM6`cIbtX#aNTq0c+Dc9?4M5hQ^DMeNx;Q&Qii)Gw^W$mcj-
zn>-S)%J~s#^*O<19EnS#e*`H}a11j4dIV{BkOEq~A5O*z5)AW1%=Nwhozlc>
znEF$@_drp_%PeCT)>B@d#TPLT*P?~ug`4Xu59){Cbdh|12PfEf%sM-sN!laO!b3}&
zb{24@eHXd2%pq0dz;B-lhuh8N;mQttM#P?Uv6>;{Ss!s$X`~SI7j7fM7@@Fs-tZ3K
zmsi%!=#iol6Zvw~j`o^}b>r)AekH%ak)3kWqDr*1nVy64-h2Hjm1y4BcNTD*!sEIi
zEVE}$Dx|ts|Cfb@hJgp^wE#;yXg_T&AHa_{CIZKC1>pgGNpUuFKvH!urk
z{~DLzTi8QlHKt}5F-nl(1i{X`i6A~@f@@dj^S)LLc+r)Ejq5)tYt#i3w|lr+LLx6Y
zvt4x+zz)iguS_aKDT>3juRG!gSiNl0X~7n((Yb<0KCu5T8ZxroReqSXXml`*yBKr&9cs
zQ{F3SwoW*T&bWFoY$xosbh{d*S|@L3TK0?x4#YT_f&Y!gGB=@NRnYp})qQQ$!Yy5FYBJ2iQI3sx
zFM#Ix$vvGhJjofoRBXevEI-Y*l*wd4vI)Cr&{uP6*CQhj#
z*`FwluoVLZGXA$V*v_sw6tr0$!M$!(jj`@8;MrkCQrU-_A+;!$sy4|$eCJ6gpz58E
zrWx_ZWmFvuRzJIs;=X-Z;#>jOH$kC9LHp$U<0+9#E_1(34QnTM?Bf1<&_JfuvQCs;qxUe0w)?V9R27=>a@AjP_2yGH9Ux%?!fh(Zwr2
z%eYlqGuO}NC3!DNk^$1-*1f>FMqFk6s|mE{1Nqsht1ln(towUzXan85@CaJY2H
z>FW=ml%&o12b?3lzljUx-R8oZ0$R!{yQCIOQ7E{WS&OE&ucFt(hzx($5yL||C5B;m
zBwL6i@#D>cWDfOD?=&FO^ec%o)px4mn3xpE%uFosVo?_j)XnEcA;f2n5~n@viS*7@
zSI>_>7BzEnk(HRlV^z70aEq>th~N{`IIDXY!j?zO3VW*N-qnjRF!{bV{_b*NrevLu
zA;OI)uduCumd;TYm97~QVDbb~czQY_wY0L*+(@%*2Q<+VL#oloFEXF<=SwyttL^y`w&!s4N=
zz^u0u*p9tqssc69Ji0q&S7Yb#>Fa}>-qcjLh&$arOe@1#n7T3xP-QX;y|Y0O;z_=w
zd8ZRi`g(&6QSS(eZ0ovs;-j4XSCGDmy}r<+jxq_b-5v
zMDmOaU$YcV@o=gOxwTiRo+0h7`N)7tB!n#eT0YCZfc6ZjJ}#t#C6}6;cjKpB5uH6_
zX0}|-x`)a6R+Z)g*CwTJ`w4rm>nQ9;ly-^Q#|b~{8R#MtiYWZdAg5j&Zdm{BYQN@g
z_6Fo8qpbZUzxB!St3I7=TXJw}Te?kKFJtqJbnhwyY3D0Kv4*w}yza6P7E?w1U7EZ&
zq$w&DY
z0B+?DQvyAK8g6frQ%qH6=n(XjPZf?1b7|fe8`cX1PU&y5)+1y`SK7Uawl6w?AKh4<
z3%%PwoIpUk4&-WKCO%LdA?Zc@G=BJ0UsI2U)2{jLGric@M=nH{{h5i{{Gq4oQ4MHF
z&{CN}V3QeK>g?M^_7U9Gufrg^JR6Hbqtj030+sN#<%R3Rah9NZSkdVPXifLZPf0_Z
z-8MEd(mF4Fd>HzvtZ4zG#5LU1E_6tPECyaXj|!{~GVHXer3L4uYo<(n{L?X@gEtjO
z*sk;y*7I{k1^9D>5=k7gE%%p;PWuF*vt3?1US50x4}yBUB+aBC39>e#pbq#$GHN(#Bpw44S@)D`!VcXfwy$U=Icn%T~oo@7)vk%-K2
z9Q~-Mj73g=PK#R|j2awG9{Gc-(K}^ch$Ez6#tGv0$HTv9RT-0_<#1d56~^+l?&XzD$NADEJ@{E4szpwH3JQVhXpj
z)2Mont%OJ^%J(wkN^FhSb!CKd{`$OeFr+m*LglMr4UsWXG4g%(bUxzj#vv{e>hrZ?
zO5%_R+c=>X&A&~VpS@F_^#qDP5r7y+b+3jJu9?@?OC+7aN82N5v$Tw+S16L1x#pL7
zbd%Wqo-n#a2)$bcuEz`fczQ89!+__E-se2;>%Wo90?rkPb
zGw-WXz5`am`re1(%`eICLc@7qXBvF%R6vPr`c~?oMzE3V`2}^AxEFEOn10S+Mes{?
zBaTcsADq7fS=EZGKm(j)`Nw4IMzDm83Qr@UwYvHVYuGHDC!eE*Y8j18V_jVx;p1Z$
zh~+ABI+)Qy(=zQ<+8*ar@4A4O9hI(u-F2#%F;)#z=c6jcHhCf6xpSBQXSOMZpKXkJ
zs9fsUikA1JUu<4f!5!)gu@fk%_x5%tM-(wMGf*ro8*YZ|OG~ix{fD@jkBQ`KuhD@&
zC(yeHlKx;(uU|wrS~m+8D6JA@Fudt2`e1suh2Af+rvECBpq*RLG98RcC!?}x*5w(i
z++O)MnN!ojw%6^xL5#x0S!+teU@h~6;A4hS_%Q60mz&Dsm(~oO1Vujm*`{)pB0C?v
zC_8BgMHcWzw;8#KZvv*)=~9)@X1y!XKo9Xk`{$wPHOX*_Z5YzZCBC34!4Bo8P!@wF
z4593N{&4@AJBXUm@*2_PNkUL-`rIAdRkD12+E@yLVAzx!!lv(-`3w=u=BV8(tTquM
zPQgtFPrT!*0-+Bj{lRl|e{Y8NR6
z@Z`pC8wrl}N}Ks!dZ`PV(MxpKnk>SnJc*wE=u-)C*;GF7Gg1+^5*2W(?|jY~Dd4Z?
znvD|NbBY)70jr5?I|^F0XBGBW{h$w@$r+$IRl>sv55tbhoY0kiBWMEaQ)tSzU!5VW
zCjNEgCrF=cr};On9lY`XDEsQCw!7fn;)NC~#jTX0#XUGJQY@6>Uff-RLyHx64_4gW
z2^1&}fe^GniW8)0C|2O+{m#8J_oq8^^9N++e9rEk-RF6B*E-3)e{6i8Pix*EtG7
z;;WGNTc|<$jNc~*$jKM5skVN&E*!srk|qD)fD6LgqqX+g$%)C>@jcaZRB9jm3Hv?c
zl+FIDFYju<{-DyO=9psGT59WY;tfkIz*IdlRDaalM`jQ1)D8}{z3D(El!BZs_0bv4
zvLf4qz(aju3SmW$NT>x5ARCbu$3wFof31tvYjCUA)r=8@6V|7SVtho}o~Hwd-+%Xk
zK!)Ejm0Maap-cL6gJ2FncJ^-)lsa7Hnvz*5{g4+N4Cx{ZU%@leb}Sivf9290G4ig=
zK!1sxBQ&{3V8W@wz?hHkGwMIcx{|#RPlJz1^}b#UiE17**zg*V)YUIrv;wp;u%tE{
zdSnUscm71CS2O>e(*@CF_G7Z80QyRs1;|MVXp5TBP
z7UGRg(jyeo|G06}!#W#l_}txmEyX5~q1VqWi7g}6d>=YQVf$LAI7Alod4G8{D|8Cc
znd4IrJ|w?J)gU(*@*L$w?%7_INbFef+yVQ${fSyH=uip9+GHbEf${
zm2$s#W{9ov73=;H9@LiA6J4bSnEbLDoP$u1+$iFmM`M|-VU+K*bm4eFIYpA+TO~BZ
zSN^84=n5}TSEsDRbxxZd6XJ|y1IBWntF(C3f~+tn`29NsY`wQW=v(hF{M=vrlOE+6
z%V*M_zQam{ut}Yj;I#m{4uZs@rFW&pg9xODw4OaiOR-u|*2&ghjepE`=NpmqVo!_w
zl|5L|r5Hj&Pw7|SG-iq$h$(qq1!efy-