From 48f10cf50e82e3d7fa889bdba2ac7c3307f8a365 Mon Sep 17 00:00:00 2001 From: porteratzo <44075849+porteratzo@users.noreply.github.com> Date: Thu, 15 Feb 2024 13:58:52 -0600 Subject: [PATCH] Add ray grouped runtime (#910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix flake8 error in local runtime (#764) * Removes unnecessary dict comprehension Signed-off-by: Patrick Foley * Removes unnecessary dict comprehension Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update ROADMAP.md (#765) Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update README.md Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update GOVERNANCE.md Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update ROADMAP.md (#785) Typos Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Updated integrations to GaNDLF (#781) * renaming loader and runner Signed-off-by: sarthakpati * updated plan to pick the new names Signed-off-by: sarthakpati * new key name Signed-off-by: sarthakpati * allow the ability to pass a file to `gandlf_config_dict` in addition to fully-fledged parameters Signed-off-by: sarthakpati * checking this differently Signed-off-by: sarthakpati * rename variable for clarity Signed-off-by: sarthakpati --------- Signed-off-by: sarthakpati Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update README.md Removed references to Intel's ownship, given it's now owned by the LF AI and Data. Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Fix Flake8 C419 for Ubuntu CI (#800) C419 Unnecessary list comprehension passed to any()/all() prevents short-circuiting - rewrite as a generator Signed-off-by: Aleksander Kantak Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Introduced shard descriptor based collaborator private attributes Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Adding batch size for train, and test in config.yaml file Files modified: 1. config.yaml 2. mnist_shard_descriptor.py 3. Workflow_Interface_101_MNIST.ipynb 4. participants.py Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Introducing multiple config yaml files Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Removing unnecessary config.yaml file. Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Added collaborator private atribute delayed initialization for local_runtime Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Incorporated review comments Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Added multi-pricessing ray backend support and, aggregator yaml file Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Updated multi-processing code Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * RayExecutor class moved from participants.py to localruntime.py Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * RayExecytor moved from interface/pariticipants.py to runtime/local_runtime.py Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Added Aggregator private attribute initialation in runtime Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Removed unnecessary import statements Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Code cleaned up, validated checkpoints manually Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Refactored, and added some new doc string Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Resolved Flake8 instructions Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Recusrsion removal + Serialization removal integrated Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Incoporated Review Comments Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Removed configuration YAML files, and added functionality to initialize private attributes by calling a callback function created by end-user Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Removed commented code Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Implemented new approach, two example files given 1. Workflow_Interface_101_MNIST.py 2. Workflow_Interface_301_MNIST_Watermarking.py Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Internal Review Comments Incorporated Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * No private attributes are required If private attributes are not provided, by default take an empty dictionary no need to pass a callable function. Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Update participants.py Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Added a check for GPU Resource Allocation Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Modified error message for resource allocation Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Resolved bug found during testing phase Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Modifide all the test cases, and following tutorials 1. Privacy Meter 2. FedProx Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Added following test cases: 1. Workflow_Interface_101_MNIST.ipynb 2. Workflow_Interface_102_Aggregator_Validation.ipynb 3. Workflow_Interface_301_MNIST_Watermarking.ipynb 4. Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb 5. Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Modified and Added Global_DP tutorials. Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Modified and Added tutorial Workflow-Interface_201_Exclusive_GPUs_with_Ray.ipynb Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Modified documentation for Workflow_Interface_201 tutorial. Signed-off-by: Parth Mandaliya Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * fixed flake-8 errors Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * reverted import module code Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya * Resolved merge conflicts in local_runtime.py --------- Fix flake8 error in local runtime (#764) * Removes unnecessary dict comprehension Signed-off-by: Patrick Foley * Removes unnecessary dict comprehension Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley --------- Signed-off-by: Parth Mandaliya * Update README.md Signed-off-by: Parth Mandaliya * Fix warnings and issues in docs (#825) * Fix warnings and issues in docs Signed-off-by: Aleksander Kantak * fixup! Fix warnings and issues in docs Signed-off-by: Aleksander Kantak --------- Signed-off-by: Aleksander Kantak Signed-off-by: Parth Mandaliya * Add Logo (#827) * Add Logo * Update README.md Signed-off-by: Parth Mandaliya * Change OpenFL documentation font to improve accessibility (#809) This replaces the font of OpenFL documents with Intel One Mono font for low vision developers. Known issues: 1. The text font within the images has not been changed. 2. Some icons that do not exist in the new font cannot be displayed properly. Fixes securefederatedai#799 Co-authored-by: Wang, Le Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Co-authored-by: He, Dan H Signed-off-by: Parth Mandaliya * Update unit tests to improve code coverage (#821) * Update ci config Signed-off-by: Fang, Xiaoran * Add unit test for following files - openfl/federated/plan/plan.py - openfl/interface/aggregation_functions/core/adaptive_aggregation.py Signed-off-by: Fang, Xiaoran * Add some test cases for databases module Signed-off-by: Fang, Xiaoran * Fix bugs for databases module unittest Signed-off-by: Fang, Xiaoran * Update unit tests for component module Signed-off-by: Fang, Xiaoran * Restore workflow config and update some comments Signed-off-by: Fang, Xiaoran * Enable save_ test case. Add yaml under test dir for unit test usage. Signed-off-by: Fang, Xiaoran * Remove plan to new dir. Signed-off-by: Fang, Xiaoran * Remove plan to new dir. Signed-off-by: Fang, Xiaoran * Add aggregator start test cases. Signed-off-by: Fang, Xiaoran * Add 2 aggregator test cases. Signed-off-by: Fang, Xiaoran * Add 1 aggregator test case. Signed-off-by: Fang, Xiaoran * Format code. Signed-off-by: Fang, Xiaoran * Refactor code. Signed-off-by: Fang, Xiaoran * Add collaborator start test cases. Signed-off-by: Fang, Xiaoran * Add 1 collaborator test case. Signed-off-by: Fang, Xiaoran * Format with flake8 Signed-off-by: Fang, Xiaoran * Remove TODO comments Signed-off-by: Fang, Xiaoran --------- Signed-off-by: Fang, Xiaoran Co-authored-by: Wang, Wenjie Co-authored-by: Lei5 Chen Signed-off-by: Parth Mandaliya * Add PyTorch linear regression example (#808) This adds a new tutorial example on distributing a linear regression task over OpenFL cluster. The model is defined by Pytorch which is able to run over both cpu (by default) and gpu. The dataset is generated by make_regression from sklearn.datasets with pre-defined parameters. Fixes #797 Co-authored-by: Jiang, Jiaqiu Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Parth Mandaliya * This prints out the hash of the CSR to disk for both the aggregator and (#813) * This prints out the hash of the CSR to disk for both the aggregator and collaborator. The user then compares and approves this hash with the hash printed out of the file to validate the CSR. In addition, a warning message is pritned if certify is run in silent mode. Fixes securefederatedai#692 Signed-off-by: Grant Baker * Refactor read_csr function to use get_csr_hash Signed-off-by: Grant Baker * Ask to check hashes before prompt --------- Signed-off-by: Grant Baker Co-authored-by: Grant Baker Signed-off-by: Parth Mandaliya * Improve workspace requirements import (#810) Remove the dump_requirement_file operation in export_ method. Fixes securefederatedai#767 Co-authored-by: Li, Qingqing Co-authored-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Parth Mandaliya * Issue 506 Added Example using FedProx (#818) * created new ineractive_api dir to hold pytorch fedprox mnist example corrected files changed to FedProxOptimizer and ran set_old_weights for new FedProx Pytorch example renamed FedProx notebook used mode.parrameters() to get pytorch model weights got weights using state_dict changed old wieghts to list (for serialization) and fixed README input wieghts before zero_grad [Enhancement: 506] Add an example that uses the FedProx optimizer in the interative_api This duplicates the MedNIST_2D example in the interative api but changes it to use the FedProx optimizer. Fixes: #506 Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme * [Enhancement: 506] Add an example that uses the FedProx optimizer in the interative_api This duplicates the MedNIST_2D example in the interative api but changes it to use the FedProx optimizer. Fixes: securefederatedai#506 Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme * Update README.md Signed-off-by: Beverly Klemme * addressed comments by psfoley: corrected words in the jupyter notebook metadata and added a link to the FedProx paper in the README. Signed-off-by: Beverly Klemme --------- Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Parth Mandaliya * [Bug: 768] FX CLI: Separate create, cert gen commands (#807) This change separates existing command "fx collaborator.py generate-cert-request" command into two commands. "fx collaborator create -n {NAME} -d {DATA_PATH: optional}". "fx collaborator generate-cert-request -n {NAME}". Fixes #768 Signed-off-by: Emmanuel Jillela Co-authored-by: Emmanuel Jillela Signed-off-by: Parth Mandaliya * Add new tutorial example to OpenFL interactive API (#812) * Add new tutorial example to OpenFL interactive API This adds a new tutorial example on distributing a linear regression task over OpenFL cluster The model is defined by scikit-learn which is able to run over both cpu (by default) and gpu. The dataset is 1-dimensional noisy data of sinusoid with pre-defined parameters. Fixes #798 Co-authored-by: Beverly Klemme Co-authored-by: Grant Baker Signed-off-by: Yi CAO * reduced requirements.txt in workspace Signed-off-by: Beverly Klemme --------- Signed-off-by: Yi CAO Signed-off-by: Beverly Klemme Co-authored-by: Yi CAO Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow in /openfl-workspace/tf_cnn_histology (#776) Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.9.3 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.9.3...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow (#777) Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.9.3 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.9.3...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Parth Mandaliya * Running a federation with GaNDLF Documentation (#794) * Initial commit of Running the federation with GaNDLF Documentation Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update docs/running_the_federation_with_gandlf.rst Co-authored-by: Sarthak Pati Signed-off-by: Patrick Foley * Update README.md Removed references to Intel's ownship, given it's now owned by the LF AI and Data. Signed-off-by: Patrick Foley * Fix Flake8 C419 for Ubuntu CI (#800) C419 Unnecessary list comprehension passed to any()/all() prevents short-circuiting - rewrite as a generator Signed-off-by: Aleksander Kantak Signed-off-by: Patrick Foley * Update README.md Signed-off-by: Patrick Foley * Fix warnings and issues in docs (#825) * Fix warnings and issues in docs Signed-off-by: Aleksander Kantak * fixup! Fix warnings and issues in docs Signed-off-by: Aleksander Kantak --------- Signed-off-by: Aleksander Kantak Signed-off-by: Patrick Foley * Add Logo (#827) * Add Logo * Update README.md Signed-off-by: Patrick Foley * Change OpenFL documentation font to improve accessibility (#809) This replaces the font of OpenFL documents with Intel One Mono font for low vision developers. Known issues: 1. The text font within the images has not been changed. 2. Some icons that do not exist in the new font cannot be displayed properly. Fixes securefederatedai#799 Co-authored-by: Wang, Le Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Co-authored-by: He, Dan H Signed-off-by: Patrick Foley * Update unit tests to improve code coverage (#821) * Update ci config Signed-off-by: Fang, Xiaoran * Add unit test for following files - openfl/federated/plan/plan.py - openfl/interface/aggregation_functions/core/adaptive_aggregation.py Signed-off-by: Fang, Xiaoran * Add some test cases for databases module Signed-off-by: Fang, Xiaoran * Fix bugs for databases module unittest Signed-off-by: Fang, Xiaoran * Update unit tests for component module Signed-off-by: Fang, Xiaoran * Restore workflow config and update some comments Signed-off-by: Fang, Xiaoran * Enable save_ test case. Add yaml under test dir for unit test usage. Signed-off-by: Fang, Xiaoran * Remove plan to new dir. Signed-off-by: Fang, Xiaoran * Remove plan to new dir. Signed-off-by: Fang, Xiaoran * Add aggregator start test cases. Signed-off-by: Fang, Xiaoran * Add 2 aggregator test cases. Signed-off-by: Fang, Xiaoran * Add 1 aggregator test case. Signed-off-by: Fang, Xiaoran * Format code. Signed-off-by: Fang, Xiaoran * Refactor code. Signed-off-by: Fang, Xiaoran * Add collaborator start test cases. Signed-off-by: Fang, Xiaoran * Add 1 collaborator test case. Signed-off-by: Fang, Xiaoran * Format with flake8 Signed-off-by: Fang, Xiaoran * Remove TODO comments Signed-off-by: Fang, Xiaoran --------- Signed-off-by: Fang, Xiaoran Co-authored-by: Wang, Wenjie Co-authored-by: Lei5 Chen Signed-off-by: Patrick Foley * Add PyTorch linear regression example (#808) This adds a new tutorial example on distributing a linear regression task over OpenFL cluster. The model is defined by Pytorch which is able to run over both cpu (by default) and gpu. The dataset is generated by make_regression from sklearn.datasets with pre-defined parameters. Fixes #797 Co-authored-by: Jiang, Jiaqiu Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Patrick Foley * This prints out the hash of the CSR to disk for both the aggregator and (#813) * This prints out the hash of the CSR to disk for both the aggregator and collaborator. The user then compares and approves this hash with the hash printed out of the file to validate the CSR. In addition, a warning message is pritned if certify is run in silent mode. Fixes securefederatedai#692 Signed-off-by: Grant Baker * Refactor read_csr function to use get_csr_hash Signed-off-by: Grant Baker * Ask to check hashes before prompt --------- Signed-off-by: Grant Baker Co-authored-by: Grant Baker Signed-off-by: Patrick Foley * Improve workspace requirements import (#810) Remove the dump_requirement_file operation in export_ method. Fixes securefederatedai#767 Co-authored-by: Li, Qingqing Co-authored-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Jiang, Jiaqiu Signed-off-by: Li, Qingqing Signed-off-by: Wang, Le Signed-off-by: Wu, Caili Signed-off-by: He, Dan H Signed-off-by: Patrick Foley * Issue 506 Added Example using FedProx (#818) * created new ineractive_api dir to hold pytorch fedprox mnist example corrected files changed to FedProxOptimizer and ran set_old_weights for new FedProx Pytorch example renamed FedProx notebook used mode.parrameters() to get pytorch model weights got weights using state_dict changed old wieghts to list (for serialization) and fixed README input wieghts before zero_grad [Enhancement: 506] Add an example that uses the FedProx optimizer in the interative_api This duplicates the MedNIST_2D example in the interative api but changes it to use the FedProx optimizer. Fixes: #506 Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme * [Enhancement: 506] Add an example that uses the FedProx optimizer in the interative_api This duplicates the MedNIST_2D example in the interative api but changes it to use the FedProx optimizer. Fixes: securefederatedai#506 Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme * Update README.md Signed-off-by: Beverly Klemme * addressed comments by psfoley: corrected words in the jupyter notebook metadata and added a link to the FedProx paper in the README. Signed-off-by: Beverly Klemme --------- Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Patrick Foley * [Bug: 768] FX CLI: Separate create, cert gen commands (#807) This change separates existing command "fx collaborator.py generate-cert-request" command into two commands. "fx collaborator create -n {NAME} -d {DATA_PATH: optional}". "fx collaborator generate-cert-request -n {NAME}". Fixes #768 Signed-off-by: Emmanuel Jillela Co-authored-by: Emmanuel Jillela Signed-off-by: Patrick Foley * Add new tutorial example to OpenFL interactive API (#812) * Add new tutorial example to OpenFL interactive API This adds a new tutorial example on distributing a linear regression task over OpenFL cluster The model is defined by scikit-learn which is able to run over both cpu (by default) and gpu. The dataset is 1-dimensional noisy data of sinusoid with pre-defined parameters. Fixes #798 Co-authored-by: Beverly Klemme Co-authored-by: Grant Baker Signed-off-by: Yi CAO * reduced requirements.txt in workspace Signed-off-by: Beverly Klemme --------- Signed-off-by: Yi CAO Signed-off-by: Beverly Klemme Co-authored-by: Yi CAO Signed-off-by: Patrick Foley * build(deps): bump tensorflow in /openfl-workspace/tf_cnn_histology (#776) Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.9.3 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.9.3...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Patrick Foley * build(deps): bump tensorflow (#777) Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.9.3 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.9.3...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Patrick Foley * Update GaNDLF repo location and test CI with master branch Signed-off-by: Patrick Foley * Update GaNDLF repo location and test CI with master branch Signed-off-by: Patrick Foley * Update GaNDLF repo location and test CI with master branch Signed-off-by: Patrick Foley * Fix documentation links. Change path names and templates for CI Signed-off-by: Patrick Foley * Fix paths Signed-off-by: Patrick Foley * Fix paths Signed-off-by: Patrick Foley * Fix breaking tests Signed-off-by: Patrick Foley * Add compatible onnx version to requirements.txt file Signed-off-by: Patrick Foley * Fix wrong csv file name Signed-off-by: Patrick Foley * Fix wrong csv file name Signed-off-by: Patrick Foley * Fix wrong names in workflow file Signed-off-by: Patrick Foley * Fix wrong data path Signed-off-by: Patrick Foley * Fix lint in test_gandlf.py Signed-off-by: Patrick Foley * Fix lint errors Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Signed-off-by: Aleksander Kantak Signed-off-by: He, Dan H Signed-off-by: Fang, Xiaoran Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Emmanuel Jillela Signed-off-by: Yi CAO Co-authored-by: Sarthak Pati Co-authored-by: Prashant Shah <40899779+SprashAI@users.noreply.github.com> Co-authored-by: akantak Co-authored-by: wangleflex <106506636+wangleflex@users.noreply.github.com> Co-authored-by: He, Dan H Co-authored-by: xiaoranf Co-authored-by: Wang, Wenjie Co-authored-by: Lei5 Chen Co-authored-by: Beverly Klemme <35578090+bjklemme-intel@users.noreply.github.com> Co-authored-by: Emmanuel Jillela Co-authored-by: Yi CAO Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Parth Mandaliya * Fixed GaNDLF rst issues. Add sphinxcontrib-mermaid (#841) Signed-off-by: Patrick Foley Signed-off-by: Parth Mandaliya * Fix GaNDLF documentation links (#842) * Fixed GaNDLF rst issues. Add sphinxcontrib-mermaid Signed-off-by: Patrick Foley * Fix links in GaNDLF Documentation * Fixed GaNDLF rst issues. Add sphinxcontrib-mermaid Signed-off-by: Patrick Foley * Fix links in GaNDLF Documentation Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Signed-off-by: Parth Mandaliya * Fix incorrectly formatted link in docs (#839) Signed-off-by: Francis Storr Signed-off-by: Parth Mandaliya * Resolving merge conflicts in local-runtime.py Integrated aggregator as stateful actor branch, tested. Signed-off-by: Parth Mandaliya -------- Signed-off-by: Parth Mandaliya * build(deps): bump onnx in /openfl-workspace/gandlf_seg_test (#840) Bumps [onnx](https://github.com/onnx/onnx) from 1.12 to 1.13.0. - [Release notes](https://github.com/onnx/onnx/releases) - [Changelog](https://github.com/onnx/onnx/blob/main/docs/Changelog.md) - [Commits](https://github.com/onnx/onnx/compare/v1.12.0...v1.13.0) --- updated-dependencies: - dependency-name: onnx dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Signed-off-by: Parth Mandaliya * Merged changes of remove-torch-dependency branch Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update setup.py Upgrading protobuf to 3.20.3 as per tensorboard requirement Signed-off-by: Parth Mandaliya * Resolving merge conflicts Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Accessibility updates (#861) * Fix incorrectly formatted link in docs Signed-off-by: Francis Storr * Font styling, color contrast, other accessibility updates This update: 1. Restores Roboto and Lato fonts for most body copy, leaving Intel One Mono for code samples. 2. Adds colors (in `colors.css`) 3. Adds a new `accessibility_overrides.css` file containing CSS that improves the accessibility of the documentation and, where possible, Read The Docs. These updates remediate numerous non-conforming WCAG 2.x Level AA bugs. The use of a separate file for this hopefully makes these changes easier to manage and less likely to be accessibility overwritten in the future. Closes #848 Signed-off-by: Francis Storr --------- Signed-off-by: Francis Storr Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.8.4 to 2.11.1 in /openfl-workspace/keras_nlp (#773) * build(deps): bump tensorflow in /openfl-workspace/keras_nlp Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Patrick Foley * Update RMSProp optimizer import Signed-off-by: Patrick Foley * flake8 Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.8.4 to 2.11.1 in /openfl-workspace/keras_cnn_mnist (#771) * build(deps): bump tensorflow in /openfl-workspace/keras_cnn_mnist Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * revert experimental Adam to legacy (#863) Signed-off-by: kta-intel --------- Signed-off-by: dependabot[bot] Signed-off-by: kta-intel Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kevin Ta <116312994+kta-intel@users.noreply.github.com> Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.8.4 to 2.11.1 in /openfl-workspace/keras_cnn_with_compression (#770) * build(deps): bump tensorflow Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: Patrick Foley * Update Adam Optimizer import Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.9.3 to 2.11.1 in /openfl-tutorials/interactive_api/Flax_CNN_CIFAR (#775) * build(deps): bump tensorflow Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.9.3 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.9.3...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Fixed breaking backages Signed-off-by: Patrick Foley * Add quiet flag back to pip install Signed-off-by: Patrick Foley --------- Signed-off-by: dependabot[bot] Signed-off-by: Patrick Foley Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow-cpu from 2.8.4 to 2.11.1 in /openfl-workspace/keras_nlp_gramine_ready (#769) * build(deps): bump tensorflow-cpu Bumps [tensorflow-cpu](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow-cpu dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Use legacy RMSprop optimizer Signed-off-by: Patrick Foley --------- Signed-off-by: dependabot[bot] Signed-off-by: Patrick Foley Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Accessibility color contrast fixes (#864) * Fix incorrectly formatted link in docs Signed-off-by: Francis Storr * Font styling, color contrast, other accessibility updates This update: 1. Restores Roboto and Lato fonts for most body copy, leaving Intel One Mono for code samples. 2. Adds colors (in `colors.css`) 3. Adds a new `accessibility_overrides.css` file containing CSS that improves the accessibility of the documentation and, where possible, Read The Docs. These updates remediate numerous non-conforming WCAG 2.x Level AA bugs. The use of a separate file for this hopefully makes these changes easier to manage and less likely to be accessibility overwritten in the future. Closes #848 Signed-off-by: Francis Storr * Color contrast updates for accessibility Color contrast updates for accessibility - update generic `a` element - update color of links in the toggle-able read-the-docs panel - update the color of the text in search results - update the color of notes headers Signed-off-by: Francis Storr --------- Signed-off-by: Francis Storr Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Tweak link color so it’s not so aggressive (#865) Signed-off-by: Francis Storr Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.8.4 to 2.11.1 in /tests/github/interactive_api_director/experiments/tensorflow_mnist/envoy (#772) * build(deps): bump tensorflow Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update sd_requirements.txt * revert to legacy SGD and install tensorflow==2.11 for workflow Signed-off-by: kta-intel --------- Signed-off-by: dependabot[bot] Signed-off-by: kta-intel Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Patrick Foley Co-authored-by: kta-intel Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * build(deps): bump tensorflow from 2.8.4 to 2.11.1 in /openfl-workspace/tf_2dunet (#774) * build(deps): bump tensorflow in /openfl-workspace/tf_2dunet Bumps [tensorflow](https://github.com/tensorflow/tensorflow) from 2.8.4 to 2.11.1. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.8.4...v2.11.1) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Update requirements.txt to retrigger CI * Update requirements.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Kevin Ta <116312994+kta-intel@users.noreply.github.com> Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update Tensorflow, gRPC, Protobuf dependencies (#868) * Update Tensorflow to latest, finally update grpcio/protobuf Signed-off-by: Patrick Foley * Lint issue fix and missing tf reference Signed-off-by: Patrick Foley * pyzmq version fixed * fix taskrunner tests for windows Signed-off-by: Mansi Sharma * fix taskrunner test syntax for windows Signed-off-by: Mansi Sharma * adding user option to workspace pip install requirements for windows Signed-off-by: Mansi Sharma * fix windows CI test Signed-off-by: Mansi Sharma * testing virtual env for windows github actions Signed-off-by: Mansi Sharma * testing virtual env for windows github actions Signed-off-by: Mansi Sharma * testing virtual env for windows github actions Signed-off-by: Mansi Sharma * testing venv for windows Signed-off-by: Mansi Sharma * test venv for windows * test venv for windows * Added new KerasSerializer. Fixed other Interactive API experiments * Update taskrunner.yml * Update taskrunner.yml * Update workspace.py * Update workspace.py * Update taskrunner.yml * Remove get_model import from global namespace so dependencies are not loaded into memory unnecessarily (breaking windows build) * Refactoring and cleaning up imports to support Windows install * Fixed logger import paths * Fix missing imports * Fix native import * Fix lint errors * Fix keras optimizer patch. Remove irrelevant unit test * Format logs in UTF-8 for windows * Update interactive-kvasir.yml * Consolidate github actions python versions to single file * Update python versions * Update python versions * Update python versions * Reduce # of DataLoader workers for Pytorch Kvasir CI test * Fix Windows encoding * Fix Windows encoding and limit rounds so Github Actions CI doesn't run out of memory Signed-off-by: Patrick Foley * Fix windows encoding * Fix Windows encoding --------- Signed-off-by: Patrick Foley Signed-off-by: Mansi Sharma Co-authored-by: Mansi Sharma <77758170+mansishr@users.noreply.github.com> Co-authored-by: Mansi Sharma Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Add FL plan description to documentation (#872) * Add plan description to documentation Signed-off-by: Mansi Sharma * fix indentation Signed-off-by: Mansi Sharma * Apply suggestions from code review Co-authored-by: Patrick Foley --------- Signed-off-by: Mansi Sharma Co-authored-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Resolved flake8 issues Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * GPU Added for aggregator Fixed issue in 103 Cyclic Institutional Incremental Learning tutorial Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Resolve Coverity Issues (#874) * Fix coverity issues * Resolve remaining coverity issues Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Migrate to Ubuntu 22.04 LTS release (supported through 2027) (#875) Signed-off-by: Patrick Foley Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Updated documentation: docs/workflow_interface.rst Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Updated Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Updated Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Updated documentation Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update workflow_interface.rst Fixing typo Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Added best model and last model extraction technique in docs/workflow_interface.rst Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Added GPU for aggregator Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Resolving merge conflicts in 103 cyclic tutorial notebook fixing FedAvg in workflow interface tutorials to be compatible with latest numpy stable release (1.24.3) (#833) * fixing FedAvg averaging in order to be compatible with numpy v1.24+ Signed-off-by: kta-intel * uncommenting installations for consistency with other tutorials Signed-off-by: kta-intel * fixing 301_MNIST_Watermarking tutorial FedAvg Signed-off-by: kta-intel * fixing 301_MNIST_Watermarki ng tutorial FedAvg Signed-off-by: kta-intel * Switching to py38 kernel and clearing cell outputs Signed-off-by: kta-intel --------- Signed-off-by: kta-intel --------- Signed-off-by: Parth Mandaliya * Resolved merge conflicts in tests/github/experimental/testflow_datastore_cli.py Testflow for verifying stdout redirection to Metaflow datastore (#758) * implemented ray.wait * reverted changes back after testing * adding datastore cli test case * removed unused variables * removed stderr validation * fixed lint suggestions Signed-off-by: Parth Mandaliya * Added weighted_average aggregation function under openfl.experimental.interface.{keras,torch}.aggregation_funtions Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya * Update EdenPipeline in the documentation (#877) Signed-off-by: Amit Portnoy <1131991+amitport@users.noreply.github.com> Signed-off-by: Parth Mandaliya * WIP: CI Scans (#873) * Initial scans commit for bandit, hadolint, trivy Signed-off-by: Patrick Foley * Address bandit scan results Signed-off-by: Patrick Foley * Fix Trivy action Signed-off-by: Patrick Foley * Fix linting Signed-off-by: Patrick Foley * Add Coverity Badge Signed-off-by: Patrick Foley * Update Hadolint threshold to flag errors only Signed-off-by: Patrick Foley * Update Hadolint threshold to flag errors only Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Signed-off-by: Parth Mandaliya * Update ROADMAP.md (#878) Signed-off-by: Parth Mandaliya * initial commit * add docstrings * change importlib to import * remove unnecesary files, replace ray with ray_grouped * remove max concurency, add number of actors * Trigger CI * run tests * lint changes * flake * changed number of actors to num_actors, added docs * Fixed workflow API tests Signed-off-by: Patrick Foley * lint fixes Signed-off-by: Patrick Foley --------- Signed-off-by: Patrick Foley Signed-off-by: ParthM-GitHub Signed-off-by: Parth Mandaliya Signed-off-by: sarthakpati Signed-off-by: Aleksander Kantak Signed-off-by: Parth Mandaliya Signed-off-by: Parth Mandaliya Signed-off-by: He, Dan H Signed-off-by: Fang, Xiaoran Signed-off-by: Grant Baker Signed-off-by: Klemme, Beverly Signed-off-by: Baker, Grant Signed-off-by: ELizabeth Simon, Neethu Signed-off-by: Jillela, Emmanuel Signed-off-by: Beverly Klemme Signed-off-by: Elizabeth Simon, Neethu Signed-off-by: Emmanuel Jillela Signed-off-by: Yi CAO Signed-off-by: dependabot[bot] Signed-off-by: Francis Storr Signed-off-by: kta-intel Signed-off-by: Mansi Sharma Signed-off-by: Amit Portnoy <1131991+amitport@users.noreply.github.com> Co-authored-by: Patrick Foley Co-authored-by: Olga Perepelkina Co-authored-by: Joe Devon <138038+joedevon@users.noreply.github.com> Co-authored-by: Sarthak Pati Co-authored-by: Prashant Shah <40899779+SprashAI@users.noreply.github.com> Co-authored-by: akantak Co-authored-by: Parth Mandaliya Co-authored-by: Parth Mandaliya Co-authored-by: Parth Mandaliya Co-authored-by: Keerti Talwar Co-authored-by: KeertiX Co-authored-by: wangleflex <106506636+wangleflex@users.noreply.github.com> Co-authored-by: He, Dan H Co-authored-by: xiaoranf Co-authored-by: Wang, Wenjie Co-authored-by: Lei5 Chen Co-authored-by: Beverly Klemme <35578090+bjklemme-intel@users.noreply.github.com> Co-authored-by: Grant Baker Co-authored-by: Emmanuel Jillela Co-authored-by: Yi CAO Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Sarthak Pati Co-authored-by: Francis Storr Co-authored-by: Patrick Foley Co-authored-by: Kevin Ta <116312994+kta-intel@users.noreply.github.com> Co-authored-by: kta-intel Co-authored-by: Mansi Sharma <77758170+mansishr@users.noreply.github.com> Co-authored-by: Mansi Sharma Co-authored-by: Sachin Gupta Co-authored-by: Keerti Prakash Talwar <115972088+KeertiX@users.noreply.github.com> Co-authored-by: Amit Portnoy <1131991+amitport@users.noreply.github.com> --- .gitignore | 1 - docs/workflow_interface.rst | 189 ++- ...rkflow_Interface_Mnist_Implementation_1.py | 95 +- ...rkflow_Interface_Mnist_Implementation_2.py | 82 +- .../Global_DP/requirements_global_dp.txt | 2 +- .../experimental/Privacy_Meter/cifar10_PM.py | 87 +- .../requirements_privacy_meter.txt | 4 +- .../Workflow_Interface_VFL_Two_Party.ipynb | 58 +- .../Workflow_Interface_Vertical_FL.ipynb | 39 +- .../Workflow_Interface_101_MNIST.ipynb | 182 ++- ..._Interface_102_Aggregator_Validation.ipynb | 79 +- ...c_Institutional_Incremental_Learning.ipynb | 173 ++- ...w_Interface_104_Keras_MNIST_with_GPU.ipynb | 363 ++++++ ...nterface_201_Exclusive_GPUs_with_Ray.ipynb | 122 +- ...low_Interface_301_MNIST_Watermarking.ipynb | 113 +- ...ce_401_FedProx_with_Synthetic_nonIID.ipynb | 822 +++++++++++++ .../requirements_workflow_interface.txt | 9 +- openfl/experimental/interface/__init__.py | 4 +- openfl/experimental/interface/fl_spec.py | 189 ++- .../experimental/interface/keras/__init__.py | 7 + .../keras/aggregation_functions/__init__.py | 7 + .../aggregation_functions/weighted_average.py | 13 + openfl/experimental/interface/participants.py | 383 ++++-- .../experimental/interface/torch/__init__.py | 7 + .../torch/aggregation_functions/__init__.py | 7 + .../aggregation_functions/weighted_average.py | 77 ++ openfl/experimental/placement/__init__.py | 4 +- openfl/experimental/placement/placement.py | 99 +- openfl/experimental/runtime/__init__.py | 3 +- openfl/experimental/runtime/local_runtime.py | 1078 +++++++++++++---- openfl/experimental/utilities/__init__.py | 11 +- openfl/experimental/utilities/exceptions.py | 24 +- .../experimental/utilities/metaflow_utils.py | 2 - openfl/experimental/utilities/resources.py | 36 +- .../experimental/utilities/runtime_utils.py | 69 +- .../experimental/utilities/stream_redirect.py | 3 + openfl/experimental/utilities/ui.py | 7 +- openfl/experimental/utilities/utils.py | 13 - setup.py | 4 + ...ements_experimental_localruntime_tests.txt | 5 + .../experimental/testflow_datastore_cli.py | 41 +- tests/github/experimental/testflow_exclude.py | 39 +- tests/github/experimental/testflow_include.py | 38 +- .../experimental/testflow_include_exclude.py | 24 +- .../experimental/testflow_internalloop.py | 24 +- .../testflow_privateattributes.py | 58 +- .../github/experimental/testflow_reference.py | 101 +- .../testflow_reference_with_exclude.py | 162 +-- .../testflow_reference_with_include.py | 154 +-- .../testflow_subset_of_collaborators.py | 40 +- 50 files changed, 3743 insertions(+), 1410 deletions(-) create mode 100644 openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb create mode 100644 openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb create mode 100644 openfl/experimental/interface/keras/__init__.py create mode 100644 openfl/experimental/interface/keras/aggregation_functions/__init__.py create mode 100644 openfl/experimental/interface/keras/aggregation_functions/weighted_average.py create mode 100644 openfl/experimental/interface/torch/__init__.py create mode 100644 openfl/experimental/interface/torch/aggregation_functions/__init__.py create mode 100644 openfl/experimental/interface/torch/aggregation_functions/weighted_average.py delete mode 100644 openfl/experimental/utilities/utils.py create mode 100644 tests/github/experimental/requirements_experimental_localruntime_tests.txt diff --git a/.gitignore b/.gitignore index 8c0419b1a9..578b6ed112 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ venv/* .idea *_pb2.py *_pb2_grpc.py - *.jpg *.crt *.key diff --git a/docs/workflow_interface.rst b/docs/workflow_interface.rst index 53feb71efd..0886aa3fc9 100644 --- a/docs/workflow_interface.rst +++ b/docs/workflow_interface.rst @@ -146,30 +146,60 @@ A :code:`Runtime` defines where the flow will be executed, who the participants .. code-block:: python - # Setup participants - aggregator = Aggregator() - aggregator.private_attributes = {} - - # Setup collaborators with private attributes - collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore'] - collaborators = [Collaborator(name=name) for name in collaborator_names] - for idx, collaborator in enumerate(collaborators): - local_train = deepcopy(mnist_train) - local_test = deepcopy(mnist_test) - local_train.data = mnist_train.data[idx::len(collaborators)] - local_train.targets = mnist_train.targets[idx::len(collaborators)] - local_test.data = mnist_test.data[idx::len(collaborators)] - local_test.targets = mnist_test.targets[idx::len(collaborators)] - collaborator.private_attributes = { - 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True), - 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True) + # Aggregator + aggregator_ = Aggregator() + + collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] + + def callable_to_initialize_collaborator_private_attributes(index, n_collaborators, batch_size, train_dataset, test_dataset): + train = deepcopy(train_dataset) + test = deepcopy(test_dataset) + train.data = train_dataset.data[index::n_collaborators] + train.targets = train_dataset.targets[index::n_collaborators] + test.data = test_dataset.data[index::n_collaborators] + test.targets = test_dataset.targets[index::n_collaborators] + + return { + "train_loader": torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True), + "test_loader": torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=True), } - # This is equivalent to: - # local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='single_process') - local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) - -Let's break this down, starting with the :code:`Aggregator` and :code:`Collaborator` placeholders. These placeholders represent the nodes where tasks will be executed. Each participant placeholder has its own set of :code:`private_attributes`; a dictionary where the key is the name of the attribute, and the value is the object. In the above example, each of the four collaborators ('Portland', 'Seattle', 'Chandler', and 'Bangalore'), have a :code:`train_loader` and `test_loader` that they can access. These private attributes can be named anything, and do not necessarily need to be the same across each participant. + # Setup collaborators private attributes via callable function + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + index=idx, + n_collaborators=len(collaborator_names), + train_dataset=mnist_train, + test_dataset=mnist_test, + batch_size=64 + ) + ) + + local_runtime = LocalRuntime(aggregator=aggregator_, collaborators=collaborators) + +Let's break this down, starting with the :code:`Aggregator` and :code:`Collaborator` components. These components represent the *Participants* in a Federated Learning experiment. Each participant has its own set of *private attributes* that represent the information / data specific to its role or requirements. As the name suggests these *private attributes* are accessible only to the particular participant, and are appropriately inserted into or filtered out of current Flow state when transferring from between Participants. For e.g. Collaborator private attributes are inserted into :code:`flow` when transitioning from Aggregator to Collaborator and are filtered out when transitioning from Collaborator to Aggregator. + +In the above :code:`FederatedFlow`, each collaborator accesses train and test datasets via *private attributes* :code:`train_loader` and :code:`test_loader`. These *private attributes* need to be set using a (user defined) callback function while instantiating the participant. Participant *private attributes* are returned by the callback function in form of a dictionary, where the key is the name of the attribute and the value is the object. + +In this example callback function :code:`callable_to_initialize_collaborator_private_attributes()` returns the collaborator private attributes :code:`train_loader` and :code:`test_loader` that are accessed by collaborator steps (:code:`aggregated_model_validation`, :code:`train` and :code:`local_model_validation`). Some important points to remember while creating callback function and private attributes are: + + - Callback Function needs to be defined by the user and should return the *private attributes* required by the participant in form of a key/value pair + - In above example multiple collaborators have the same callback function. Depending on the Federated Learning requirements, user can specify unique callback functions for each Participant + - If no Callback Function is specified then the Participant shall not have any *private attributes* + - Callback function can be provided with any parameters required as arguments. In this example, parameters essential for the callback function are supplied with corresponding values bearing *same names* during the instantiation of the Collaborator + + * :code:`index`: Index of the particular collaborator needed to shard the dataset + * :code:`n_collaborators`: Total number of collaborators in which the dataset is sharded + * :code:`batch_size`: For the train and test loaders + * :code:`train_dataset`: Train Dataset to be sharded between n_collaborators + * :code:`test_dataset`: Test Dataset to be sharded between n_collaborators + + - Callback function needs to be specified by user while instantiating the participant. Callback function is invoked by the OpenFL runtime at the time participant is created and once created these attributes cannot be modified + - Private attributes are accessible only in the Participant steps Now let's see how the runtime for a flow is assigned, and the flow gets run: @@ -184,23 +214,43 @@ And that's it! This will run an instance of the :code:`FederatedFlow` on a singl Runtime Backends ================ -The Runtime defines where code will run, but the Runtime has a :code:`Backend` - which defines the underlying implementation of *how* the flow will be executed. :code:`'single_process'` is the default in the :code:`LocalRuntime`: it executes all code sequentially within a single python process, and is well suited to run both on high spec and low spec hardware. For users with large servers or multiple GPUs they wish to take advantage of, we also provide a `Ray ` backend. The Ray backend enables parallel task execution for collaborators, and optionally allows users to request dedicated GPUs for collaborator tasks in the placement decorator, as follows: +The Runtime defines where code will run, but the Runtime has a :code:`Backend` - which defines the underlying implementation of *how* the flow will be executed. :code:`single_process` is the default in the :code:`LocalRuntime`: it executes all code sequentially within a single python process, and is well suited to run both on high spec and low spec hardware + +For users with large servers or multiple GPUs they wish to take advantage of, we also provide a :code:`ray` `` backend. The Ray backend enables parallel task execution for collaborators, and optionally allows users to request dedicated CPU / GPUs for Participants by using the :code:`num_cpus` and :code:`num_gpus` arguments while instantiating the Participant in following manner: .. code-block:: python - ExampleDedicatedGPUFlow(FLSpec): - ... - # We request one dedicated GPU for this task - @collaborator(num_gpus=1) - def training(self): - print(f'CUDA_VISIBLE_DEVICES: {os.environ["CUDA_VISIBLE_DEVICES"]}')) - self.loss = train_func(self.model, self.train_loader) - self.next(self.validation) - ... - + # Aggregator + aggregator_ = Aggregator(num_gpus=0.2) + + collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] + + def callable_to_initialize_collaborator_private_attributes(index, n_collaborators, batch_size, train_dataset, test_dataset): + ... + + # Setup collaborators private attributes via callable function + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + num_gpus=0.2, # Number of the GPU allocated to Participant + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + index=idx, + n_collaborators=len(collaborator_names), + train_dataset=mnist_train, + test_dataset=mnist_test, + batch_size=64 + ) + ) + # The Ray Backend will now be used for local execution local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='ray') +In the above example, we have used :code:`num_gpus=0.2` while instantiating Aggregator and Collaborator to specify that each participant shall use 1/5th of GPU - this results in one GPU being dedicated for a total of 4 collaborators and 1 Aggregator. Users can tune these arguments based on their Federated Learning requirements and available hardware resources. Configurations where one Participant is shared across GPUs is not supported. For e.g. trying to run 5 participants on 2 GPU hardware with :code:`num_gpus=0.4` will not work since 80% of each GPU is allocated to 4 participants and 5th participant does not have any available GPU remaining for use. + +**Note:** It is not necessary to have ALL the participants use GPUs. For e.g. only the Collaborator are allocated to GPUs. In this scenario user should ensure that the artifacts returned by Collaborators to Aggregator (e.g. locally trained model object) should be loaded back to CPU before exiting the collaborator step (i.e. before the join step). As Tensorflow manages the object allocation by default therefore this step is needed only for Pytorch. + Debugging with the Metaflow Client ================================== @@ -218,19 +268,38 @@ After the flow has started running, you can use the Metaflow Client to get inter .. code-block:: python - from metaflow import Flow, Run, Task, Step - + from metaflow import Metaflow, Flow, Step, Task + + # Initialize Metaflow object and obtain list of executed flows: + m = Metaflow() + list(m) + > [Flow('FederatedFlow'), Flow('AggregatorValidationFlow'), Flow('FederatedFlow_MNIST_Watermarking')] + # The name of the flow is the name of the class - flow = Flow('FederatedFlow') - run = flow.latest_run + # Identify the Flow name + flow_name = 'FederatedFlow' + + # List all instances of Federatedflow executed under distinct run IDs + flow = Flow(flow_name) + list(flow) + > [Run('FederatedFlow/1692946840822001'), + Run('FederatedFlow/1692946796234386'), + Run('FederatedFlow/1692902602941163'), + Run('FederatedFlow/1692902559123920'),] + + # To Retrieve the latest run of the Federatedflow + run = Flow(flow_name).latest_run + print(run) + > Run('FederatedFlow/1692946840822001') + list(run) - > [Step('FederatedFlow/1671152854447797/end'), - Step('FederatedFlow/1671152854447797/join'), - Step('FederatedFlow/1671152854447797/local_model_validation'), - Step('FederatedFlow/1671152854447797/train'), - Step('FederatedFlow/1671152854447797/aggregated_model_validation'), - Step('FederatedFlow/1671152854447797/start')] - step = Step('FederatedFlow/1671152854447797/aggregated_model_validation') + > [Step('FederatedFlow/1692946840822001/end'), + Step('FederatedFlow/1692946840822001/join'), + Step('FederatedFlow/1692946840822001/local_model_validation'), + Step('FederatedFlow/1692946840822001/train'), + Step('FederatedFlow/1692946840822001/aggregated_model_validation'), + Step('FederatedFlow/1692946840822001/start')] + step = Step('FederatedFlow/1692946840822001/aggregated_model_validation') for task in step: if task.data.input == 'Portland': print(task.data) @@ -260,6 +329,37 @@ And if we wanted to get log or error message for that task, you can just run: print(portland_task.stderr) > [No output] +Also, If we wanted to get the best model and the last model, you can just run: + +.. code-block:: python + + # Choose the specific step containing the desired models (e.g., 'join' step): + step = Step('FederatedFlow/1692946840822001/join') + list(step) + > [Task('FederatedFlow/1692946840822001/join/12'),--> Round 3 + Task('FederatedFlow/1692946840822001/join/9'), --> Round 2 + Task('FederatedFlow/1692946840822001/join/6'), --> Round 1 + Task('FederatedFlow/1692946840822001/join/3')] --> Round 0 + + """The sequence of tasks represents each round, with the most recent task corresponding to the final round and the preceding tasks indicating the previous rounds + in chronological order. + To determine the best model, analyze the command line logs and model accuracy for each round. Then, provide the corresponding task ID associated with that Task""" + task = Task('FederatedFlow/1692946840822001/join/9') + + # Access the best model and its associated data + best_model = task.data.model + best_local_model_accuracy = task.data.local_model_accuracy + best_aggregated_model_accuracy = t.data.aggregated_model_accuracy + + # To retrieve the last model, select the most recent Task i.e last round. + task = Task('FederatedFlow/1692946840822001/join/12') + last_model = task.data.model + + # Save the chosen models using a suitable framework (e.g., PyTorch in this example): + import torch + torch.save(last_model.state_dict(), PATH) + torch.save(best_model.state_dict(), PATH) + While this information is useful for debugging, depending on your workflow it may require significant disk space. For this reason, `checkpoint` is disabled by default. Runtimes: Future Plans @@ -279,4 +379,3 @@ Our goal is to make it a one line change to configure where and how a flow is ex federated_runtime = FederatedRuntime(...) flow.runtime = federated_runtime flow.run() - diff --git a/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_1.py b/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_1.py index eae51d76c3..8b9dbf4722 100644 --- a/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_1.py +++ b/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_1.py @@ -39,7 +39,7 @@ # Fixing the seed for result repeatation: remove below to stop repeatable runs # ---------------------------------- random_seed = 5495300300540669060 -g_device = torch.Generator(device='cuda') +g_device = torch.Generator(device="cuda") # Uncomment the line below to use g_cpu if not using cuda # g_device = torch.Generator() # noqa: E800 # NOTE: remove below to stop repeatable runs @@ -93,7 +93,6 @@ def forward(self, x): def default_optimizer(model): - """ Return a new optimizer: we have only tested torch.optim.SGD w/ momentum however, we encouraging users to test other optimizers (i.e. torch.optim.Adam) @@ -106,7 +105,6 @@ def default_optimizer(model): def FedAvg(models, previous_global_model=None, dp_params=None): # NOQA: N802 - """ Return a Federated average model based on Fedavg algorithm: H. B. Mcmahan, E. Moore, D. Ramage, S. Hampson, and B. A. Y.Arcas, @@ -152,7 +150,7 @@ def FedAvg(models, previous_global_model=None, dp_params=None): # NOQA: N802 if len(state_dicts) > 1: for key in models[0].state_dict(): state_dict[key] = np.sum( - [state[key] for state in state_dicts], axis=0 + np.array([state[key] for state in state_dicts], dtype=object), axis=0 ) / len(models) new_model.load_state_dict(state_dict) return new_model @@ -181,7 +179,6 @@ def inference(network, test_loader, device): def optimizer_to_device(optimizer, device): - """ Sending the "torch.optim.Optimizer" object into the specified device for model training and inference @@ -244,7 +241,6 @@ def clip_testing_on_optimizer_parameters( def validate_dp_params(dp_params): - """ The differential privacy block should have the exact keys as provided below. @@ -275,7 +271,6 @@ def validate_dp_params(dp_params): def parse_config(config_path): - """ Parse "test_config.yml". @@ -288,7 +283,6 @@ def parse_config(config_path): def add_noise_on_aggegated_parameters(collaborators, model, dp_params): - """ Adds noise on aggregated model parameters performed at the aggregator. @@ -404,10 +398,7 @@ def start(self): print(f"No collaborator selected for training at Round: {self.round}") self.next(self.check_round_completion) - # Uncomment this if you don't have GPU in the machine and - # want this application to run on CPU instead - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def aggregated_model_validation(self): print(f"Performing aggregated model validation for collaborator {self.input}") self.model = self.model.to(self.device) @@ -422,10 +413,7 @@ def aggregated_model_validation(self): self.collaborator_name = self.input self.next(self.train) - # Uncomment this if you don't have GPU in the machine - # and want this application to run on CPU instead - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def train(self): print(f"Performing model training for collaborator {self.input}") self.optimizer = ClipOptimizer( @@ -490,10 +478,7 @@ def train(self): torch.cuda.empty_cache() self.next(self.local_model_validation) - # Uncomment this if you don't have GPU in the machine - # and want this application to run on CPU instead - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def local_model_validation(self): print(f"Performing local model validation for collaborator {self.input}") self.local_validation_score = inference( @@ -615,7 +600,6 @@ def end(self): if __name__ == "__main__": - argparser = argparse.ArgumentParser(description=__doc__) argparser.add_argument( "--config_path", help="Absolute path to the flow configuration file" @@ -628,11 +612,16 @@ def end(self): args = argparser.parse_args() + if torch.cuda.is_available(): + device = torch.device("cuda:0") + else: + device = torch.device("cpu") + # Setup participants - aggregator = Aggregator() - aggregator.private_attributes = {} + # Set `num_gpus=0.09` to `num_gpus=0.0` in order to run this tutorial on CPU + aggregator = Aggregator(num_gpus=0.09) - # Setup collaborators with private attributes + # Collaborator names collaborator_names = [ "Portland", "Seattle", @@ -645,34 +634,46 @@ def end(self): "CostaRica", "Guadalajara", ] - collaborators = [Collaborator(name=name) for name in collaborator_names] - - if torch.cuda.is_available(): - device = torch.device( - "cuda:0" - ) # This will enable Ray library to reserve available GPU(s) for the task - else: - # Uncomment appropriate collaborator decorators in FederatedFlow class if - # you want the application to run on CPU - device = torch.device("cpu") - for idx, collab in enumerate(collaborators): - local_train = deepcopy(mnist_train) - local_test = deepcopy(mnist_test) - local_train.data = mnist_train.data[idx::len(collaborators)] - local_train.targets = mnist_train.targets[idx::len(collaborators)] - local_test.data = mnist_test.data[idx::len(collaborators)] - local_test.targets = mnist_test.targets[idx::len(collaborators)] - collab.private_attributes = { - "train_loader": DataLoader( - local_train, batch_size=batch_size_train, shuffle=True + def callable_to_initialize_collaborator_private_attributes( + index, n_collaborators, batch_size, train_dataset, test_dataset + ): + train = deepcopy(train_dataset) + test = deepcopy(test_dataset) + train.data = train_dataset.data[index::n_collaborators] + train.targets = train_dataset.targets[index::n_collaborators] + test.data = test_dataset.data[index::n_collaborators] + test.targets = test_dataset.targets[index::n_collaborators] + + return { + "train_loader": torch.utils.data.DataLoader( + train, batch_size=batch_size, shuffle=True ), - "test_loader": DataLoader( - local_test, batch_size=batch_size_train, shuffle=True + "test_loader": torch.utils.data.DataLoader( + test, batch_size=batch_size, shuffle=True ), } - local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + # Set `num_gpus=0.09` to `num_gpus=0.0` in order to run this tutorial on CPU + num_cpus=0.0, + num_gpus=0.09, # Assuming GPU(s) is available in the machine + index=idx, + n_collaborators=len(collaborator_names), + batch_size=batch_size_train, + train_dataset=mnist_train, + test_dataset=mnist_test, + ) + ) + + local_runtime = LocalRuntime( + aggregator=aggregator, collaborators=collaborators, backend="ray" + ) print(f"Local runtime collaborators = {local_runtime.collaborators}") top_model_accuracy = 0 diff --git a/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_2.py b/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_2.py index 88dab0269d..e9f7a16a1b 100644 --- a/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_2.py +++ b/openfl-tutorials/experimental/Global_DP/Workflow_Interface_Mnist_Implementation_2.py @@ -35,7 +35,7 @@ random_seed = 5495300300540669060 -g_device = torch.Generator(device='cuda') +g_device = torch.Generator(device="cuda") # Uncomment the line below to use g_cpu if not using cuda # g_device = torch.Generator() # noqa: E800 # NOTE: remove below to stop repeatable runs @@ -400,10 +400,7 @@ def start(self): self.round += 1 self.next(self.start) - # Uncomment below line if you are using the ray backend and - # do not have a GPU accessible - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def aggregated_model_validation(self): print(f"Performing aggregated model validation for collaborator {self.input}") self.model = self.model.to(self.device) @@ -418,10 +415,7 @@ def aggregated_model_validation(self): self.collaborator_name = self.input self.next(self.train) - # Uncomment below line if you are using the ray backend and - # do not have a GPU accessible - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def train(self): print(f"Performing model training for collaborator {self.input}") @@ -487,10 +481,7 @@ def train(self): torch.cuda.empty_cache() self.next(self.local_model_validation) - # Uncomment below line if you are using the ray backend and - # do not have a GPU accessible - # @collaborator - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def local_model_validation(self): print(f"Performing local model validation for collaborator {self.input}") self.local_validation_score = inference( @@ -588,7 +579,6 @@ def end(self): if __name__ == "__main__": - argparser = argparse.ArgumentParser(description=__doc__) argparser.add_argument( "--config_path", help="Absolute path to the flow configuration file." @@ -601,9 +591,14 @@ def end(self): args = argparser.parse_args() + if torch.cuda.is_available(): + device = torch.device("cuda:0") + else: + device = torch.device("cpu") + # Setup participants - aggregator = Aggregator() - aggregator.private_attributes = {} + # Set `num_gpus=0.09` to `num_gpus=0.0` in order to run this tutorial on CPU + aggregator = Aggregator(num_gpus=0.09) # Setup collaborators with private attributes collaborator_names = [ @@ -618,35 +613,48 @@ def end(self): "CostaRica", "Guadalajara", ] - collaborators = [Collaborator(name=name) for name in collaborator_names] - if torch.cuda.is_available(): - device = torch.device( - "cuda:0" - ) # This will enable Ray library to reserve available GPU(s) for the task - else: - # Uncomment appropriate collaborator decorators in FederatedFlow class if - # you want the application to run on CPU - device = torch.device("cpu") - - for idx, collab in enumerate(collaborators): - local_train = deepcopy(mnist_train) - local_test = deepcopy(mnist_test) - local_train.data = mnist_train.data[idx::len(collaborators)] - local_train.targets = mnist_train.targets[idx::len(collaborators)] - local_test.data = mnist_test.data[idx::len(collaborators)] - local_test.targets = mnist_test.targets[idx::len(collaborators)] - collab.private_attributes = { + def callable_to_initialize_collaborator_private_attributes( + index, n_collaborators, batch_size, train_dataset, test_dataset + ): + train = deepcopy(train_dataset) + test = deepcopy(test_dataset) + train.data = train_dataset.data[index::n_collaborators] + train.targets = train_dataset.targets[index::n_collaborators] + test.data = test_dataset.data[index::n_collaborators] + test.targets = test_dataset.targets[index::n_collaborators] + + return { "train_loader": torch.utils.data.DataLoader( - local_train, batch_size=batch_size_train, shuffle=True + train, batch_size=batch_size, shuffle=True ), "test_loader": torch.utils.data.DataLoader( - local_test, batch_size=batch_size_train, shuffle=True + test, batch_size=batch_size, shuffle=True ), } - local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + # Set `num_gpus=0.09` to `num_gpus=0.0` in order to run this tutorial on CPU + num_cpus=0.0, + num_gpus=0.09, # Assuming GPU(s) is available in the machine + index=idx, + n_collaborators=len(collaborator_names), + batch_size=batch_size_train, + train_dataset=mnist_train, + test_dataset=mnist_test, + ) + ) + + local_runtime = LocalRuntime( + aggregator=aggregator, collaborators=collaborators, backend="ray" + ) print(f"Local runtime collaborators = {local_runtime.collaborators}") + best_model = None initial_model = Net() top_model_accuracy = 0 diff --git a/openfl-tutorials/experimental/Global_DP/requirements_global_dp.txt b/openfl-tutorials/experimental/Global_DP/requirements_global_dp.txt index 1a1c19a177..fafb9bf8eb 100644 --- a/openfl-tutorials/experimental/Global_DP/requirements_global_dp.txt +++ b/openfl-tutorials/experimental/Global_DP/requirements_global_dp.txt @@ -1,6 +1,6 @@ numpy==1.23.3 torch==1.13.1 -torchvision==0.13.1 +torchvision==0.14.1 opacus==1.2.0 matplotlib==3.6.0 pillow==10.2.0 diff --git a/openfl-tutorials/experimental/Privacy_Meter/cifar10_PM.py b/openfl-tutorials/experimental/Privacy_Meter/cifar10_PM.py index 84fb8ecd56..8a185065f2 100644 --- a/openfl-tutorials/experimental/Privacy_Meter/cifar10_PM.py +++ b/openfl-tutorials/experimental/Privacy_Meter/cifar10_PM.py @@ -146,7 +146,6 @@ def inference(network, test_loader, device): def optimizer_to_device(optimizer, device): - """ Sending the "torch.optim.Optimizer" object into the specified device for model training and inference @@ -345,8 +344,7 @@ def start(self): exclude=["private"], ) - # @collaborator # Uncomment if you want ro run on CPU - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def aggregated_model_validation(self): print( ( @@ -359,8 +357,7 @@ def aggregated_model_validation(self): self.collaborator_name = self.input self.next(self.train) - # @collaborator # Uncomment if you want ro run on CPU - @collaborator(num_gpus=1) # Assuming GPU(s) is available on the machine + @collaborator def train(self): print(20 * "#") print( @@ -417,8 +414,7 @@ def train(self): torch.cuda.empty_cache() self.next(self.local_model_validation) - # @collaborator # Uncomment if you want ro run on CPU - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def local_model_validation(self): print( ( @@ -456,8 +452,7 @@ def local_model_validation(self): else: self.next(self.join, exclude=["training_completed"]) - # @collaborator # Uncomment if you want ro run on CPU - @collaborator(num_gpus=1) # Assuming GPU(s) is available in the machine + @collaborator def audit(self): print( ( @@ -585,7 +580,6 @@ def end(self): if __name__ == "__main__": - argparser = argparse.ArgumentParser(description=__doc__) argparser.add_argument( "--audit_dataset_ratio", @@ -662,27 +656,24 @@ def end(self): args = argparser.parse_args() # Setup participants - aggregator = Aggregator() - aggregator.private_attributes = {} + # Set `num_gpus=0.0` to `num_gpus=0.3` to run on GPU + aggregator = Aggregator(num_gpus=0.0) - # Setup collaborators with private attributes collaborator_names = ["Portland", "Seattle"] - collaborators = [Collaborator(name=name) for name in collaborator_names] if torch.cuda.is_available(): - device = torch.device( - "cuda:0" - ) # This will enable Ray library to reserve available GPU(s) for the task + device = torch.device("cuda:0") else: device = torch.device("cpu") + # Download and setup the train, and test dataset transform = transforms.Compose([transforms.ToTensor()]) cifar_train = CIFAR10(root="./data", train=True, download=True, transform=transform) cifar_test = CIFAR10(root="./data", train=False, download=True, transform=transform) - # split the dataset + # Split the dataset in train, test, and population dataset N_total_samples = len(cifar_test) + len(cifar_train) train_dataset_size = int(N_total_samples * args.train_dataset_ratio) test_dataset_size = int(N_total_samples * args.test_dataset_ratio) @@ -699,9 +690,9 @@ def end(self): train_dataset.targets = Y[:train_dataset_size] test_dataset = deepcopy(cifar_test) - test_dataset.data = X[train_dataset_size:train_dataset_size + test_dataset_size] + test_dataset.data = X[train_dataset_size: train_dataset_size + test_dataset_size] test_dataset.targets = Y[ - train_dataset_size:train_dataset_size + test_dataset_size + train_dataset_size: train_dataset_size + test_dataset_size ] population_dataset = deepcopy(cifar_test) @@ -717,22 +708,25 @@ def end(self): ) ) - # partition the dataset for clients - for idx, collab in enumerate(collaborators): - + # Split train, test, and population dataset among collaborators + # this function will be called before executing collaborator steps + # which will return private attributes dictionary for each collaborator + def callable_to_initialize_collaborator_private_attributes( + index, n_collaborators, train_ds, test_ds, population_ds, args + ): # construct the training and test and population dataset - local_train = deepcopy(train_dataset) - local_test = deepcopy(test_dataset) - local_population = deepcopy(population_dataset) + local_train = deepcopy(train_ds) + local_test = deepcopy(test_ds) + local_population = deepcopy(population_ds) - local_train.data = train_dataset.data[idx::len(collaborators)] - local_train.targets = train_dataset.targets[idx::len(collaborators)] + local_train.data = train_ds.data[index::n_collaborators] + local_train.targets = train_ds.targets[index::n_collaborators] - local_test.data = test_dataset.data[idx::len(collaborators)] - local_test.targets = test_dataset.targets[idx::len(collaborators)] + local_test.data = test_ds.data[index::n_collaborators] + local_test.targets = test_ds.targets[index::n_collaborators] - local_population.data = population_dataset.data[idx::len(collaborators)] - local_population.targets = population_dataset.targets[idx::len(collaborators)] + local_population.data = population_ds.data[index::n_collaborators] + local_population.targets = population_ds.targets[index::n_collaborators] # initialize pm report to track the privacy loss during the training local_pm_info = PM_report( @@ -763,7 +757,7 @@ def end(self): Path(local_pm_info.log_dir).mkdir(parents=True, exist_ok=True) Path(global_pm_info.log_dir).mkdir(parents=True, exist_ok=True) - collab.private_attributes = { + return { "local_pm_info": local_pm_info, "global_pm_info": global_pm_info, "train_dataset": local_train, @@ -777,9 +771,30 @@ def end(self): ), } - # To activate the ray backend with parallel collaborator tasks run in their own process - # and exclusive GPUs assigned to tasks, set LocalRuntime with backend='ray': - local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) + collaborators = [] + for idx, collab_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collab_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + # If 1 GPU is available in the machine + # Set `num_gpus=0.0` to `num_gpus=0.3` to run on GPU + # with ray backend with 2 collaborators + num_cpus=0.0, + num_gpus=0.0, + index=idx, + n_collaborators=len(collaborator_names), + train_ds=train_dataset, + test_ds=test_dataset, + population_ds=population_dataset, + args=args, + ) + ) + + # Set backend='ray' to use ray-backend + local_runtime = LocalRuntime( + aggregator=aggregator, collaborators=collaborators, backend="single_process" + ) print(f"Local runtime collaborators = {local_runtime.collaborators}") diff --git a/openfl-tutorials/experimental/Privacy_Meter/requirements_privacy_meter.txt b/openfl-tutorials/experimental/Privacy_Meter/requirements_privacy_meter.txt index 8de0fc1660..6021e8f14c 100644 --- a/openfl-tutorials/experimental/Privacy_Meter/requirements_privacy_meter.txt +++ b/openfl-tutorials/experimental/Privacy_Meter/requirements_privacy_meter.txt @@ -1,9 +1,9 @@ torch==1.13.1 -torchvision==0.14.0 +torchvision==0.14.1 matplotlib pillow opacus==1.3.0 numpy==1.23.5 cloudpickle scikit-learn -git+https://github.com/privacytrustlab/ml_privacy_meter.git@ac181a885815f85b3809317c247f422e6596cb4a \ No newline at end of file +git+https://github.com/privacytrustlab/ml_privacy_meter.git@ac181a885815f85b3809317c247f422e6596cb4a diff --git a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb b/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb index 7e68bf0e0e..5ba5a6fbce 100644 --- a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb +++ b/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_VFL_Two_Party.ipynb @@ -150,21 +150,47 @@ "source": [ "# Setup participants\n", "aggregator = Aggregator()\n", - "aggregator.private_attributes['trainloader'] = trainloader\n", - "aggregator.private_attributes['label_model'] = label_model\n", - "aggregator.private_attributes['label_model_optimizer'] = label_model_optimizer\n", "\n", - "# Setup collaborators with private attributes\n", + "def callable_to_initialize_aggregator_private_attributes(train_loader,label_model,label_model_optimizer):\n", + " return {\"trainloader\": train_loader,\n", + " \"label_model\" : label_model,\n", + " \"label_model_optimizer\":label_model_optimizer\n", + " } \n", + "\n", + "# Setup aggregator private attributes via callable function\n", + "aggregator = Aggregator(\n", + " name=\"agg\",\n", + " private_attributes_callable=callable_to_initialize_aggregator_private_attributes,\n", + " train_loader = trainloader,\n", + " label_model=label_model,\n", + " label_model_optimizer=label_model_optimizer\n", + ")\n", + "\n", + "# Setup collaborators private attributes via callable function\n", "collaborator_names = ['Portland']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", "\n", - "for idx, collaborator in enumerate(collaborators):\n", - " collaborator.private_attributes['data_model'] = data_model\n", - " collaborator.private_attributes['data_model_optimizer'] = data_model_optimizer\n", - " collaborator.private_attributes['trainloader'] = deepcopy(trainloader)\n", + "def callable_to_initialize_collaborator_private_attributes(index,data_model,data_model_optimizer,train_loader):\n", + " return {\n", + " \"data_model\": data_model,\n", + " \"data_model_optimizer\": data_model_optimizer,\n", + " \"trainloader\" : deepcopy(train_loader)\n", + " }\n", + "\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx,\n", + " data_model = data_model,\n", + " data_model_optimizer = data_model_optimizer,\n", + " train_loader = trainloader\n", + " )\n", + " )\n", "\n", "local_runtime = LocalRuntime(\n", - " aggregator=aggregator, collaborators=collaborators, backend='single_process')\n", + " aggregator=aggregator, collaborators=collaborators, backend='ray')\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')\n", "\n", "epochs = 100\n", @@ -192,16 +218,6 @@ "run_id = vflow._run_id" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "composed-burst", - "metadata": {}, - "outputs": [], - "source": [ - "import metaflow" - ] - }, { "cell_type": "code", "execution_count": null, @@ -250,7 +266,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.16" } }, "nbformat": 4, diff --git a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb b/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb index 599d670b7a..03bd458193 100644 --- a/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb +++ b/openfl-tutorials/experimental/Vertical_FL/Workflow_Interface_Vertical_FL.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "aedbb436", "metadata": {}, @@ -122,14 +123,15 @@ "source": [ "# Setup participants\n", "aggregator = Aggregator()\n", - "aggregator.private_attributes = {}\n", "\n", - "# Setup collaborators with private attributes\n", + "# Setup collaborators private attributes via callable function\n", "collaborator_names = ['Portland', 'Seattle', 'Chandler', 'Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(Collaborator(name=collaborator_name))\n", "\n", "local_runtime = LocalRuntime(\n", - " aggregator=aggregator, collaborators=collaborators)\n", + " aggregator=aggregator, collaborators=collaborators,backend='ray')\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')\n", "\n", "vflow = VerticalFlow(checkpoint=True)\n", @@ -139,6 +141,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "308b5d35", "metadata": {}, @@ -156,16 +159,6 @@ "run_id = vflow._run_id" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "composed-burst", - "metadata": {}, - "outputs": [], - "source": [ - "import metaflow" - ] - }, { "cell_type": "code", "execution_count": null, @@ -286,22 +279,6 @@ "source": [ "t.data.round" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "auburn-working", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1ca61148", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -320,7 +297,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.16" } }, "nbformat": 4, diff --git a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb index f861522f95..156fd56dde 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "14821d97", "metadata": {}, @@ -10,6 +11,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bd059520", "metadata": {}, @@ -23,6 +25,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "39c3d86a", "metadata": {}, @@ -31,6 +34,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a7989e72", "metadata": {}, @@ -39,6 +43,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "fc8e35da", "metadata": {}, @@ -47,6 +52,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "4dbb89b6", "metadata": {}, @@ -63,14 +69,17 @@ "source": [ "!pip install git+https://github.com/intel/openfl.git\n", "!pip install -r requirements_workflow_interface.txt\n", + "!pip install torch\n", + "!pip install torchvision\n", "\n", "# Uncomment this if running in Google Colab\n", - "#!pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", - "#import os\n", - "#os.environ[\"USERNAME\"] = \"colab\"" + "# !pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", + "# import os\n", + "# os.environ[\"USERNAME\"] = \"colab\"" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7237eac4", "metadata": {}, @@ -103,19 +112,29 @@ "torch.backends.cudnn.enabled = False\n", "torch.manual_seed(random_seed)\n", "\n", - "mnist_train = torchvision.datasets.MNIST('files/', train=True, download=True,\n", - " transform=torchvision.transforms.Compose([\n", - " torchvision.transforms.ToTensor(),\n", - " torchvision.transforms.Normalize(\n", - " (0.1307,), (0.3081,))\n", - " ]))\n", + "mnist_train = torchvision.datasets.MNIST(\n", + " \"./files/\",\n", + " train=True,\n", + " download=True,\n", + " transform=torchvision.transforms.Compose(\n", + " [\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize((0.1307,), (0.3081,)),\n", + " ]\n", + " ),\n", + ")\n", "\n", - "mnist_test = torchvision.datasets.MNIST('files/', train=False, download=True,\n", - " transform=torchvision.transforms.Compose([\n", - " torchvision.transforms.ToTensor(),\n", - " torchvision.transforms.Normalize(\n", - " (0.1307,), (0.3081,))\n", - " ]))\n", + "mnist_test = torchvision.datasets.MNIST(\n", + " \"./files/\",\n", + " train=False,\n", + " download=True,\n", + " transform=torchvision.transforms.Compose(\n", + " [\n", + " torchvision.transforms.ToTensor(),\n", + " torchvision.transforms.Normalize((0.1307,), (0.3081,)),\n", + " ]\n", + " ),\n", + ")\n", "\n", "class Net(nn.Module):\n", " def __init__(self):\n", @@ -154,6 +173,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "cd268911", "metadata": {}, @@ -217,7 +237,7 @@ "source": [ "class FederatedFlow(FLSpec):\n", "\n", - " def __init__(self, model = None, optimizer = None, rounds=3, **kwargs):\n", + " def __init__(self, model=None, optimizer=None, rounds=3, **kwargs):\n", " super().__init__(**kwargs)\n", " if model is not None:\n", " self.model = model\n", @@ -225,7 +245,7 @@ " else:\n", " self.model = Net()\n", " self.optimizer = optim.SGD(self.model.parameters(), lr=learning_rate,\n", - " momentum=momentum)\n", + " momentum=momentum)\n", " self.rounds = rounds\n", "\n", " @aggregator\n", @@ -234,12 +254,12 @@ " self.collaborators = self.runtime.collaborators\n", " self.private = 10\n", " self.current_round = 0\n", - " self.next(self.aggregated_model_validation,foreach='collaborators',exclude=['private'])\n", + " self.next(self.aggregated_model_validation, foreach='collaborators', exclude=['private'])\n", "\n", " @collaborator\n", " def aggregated_model_validation(self):\n", " print(f'Performing aggregated model validation for collaborator {self.input}')\n", - " self.agg_validation_score = inference(self.model,self.test_loader)\n", + " self.agg_validation_score = inference(self.model, self.test_loader)\n", " print(f'{self.input} value of {self.agg_validation_score}')\n", " self.next(self.train)\n", "\n", @@ -250,32 +270,35 @@ " momentum=momentum)\n", " train_losses = []\n", " for batch_idx, (data, target) in enumerate(self.train_loader):\n", - " self.optimizer.zero_grad()\n", - " output = self.model(data)\n", - " loss = F.nll_loss(output, target)\n", - " loss.backward()\n", - " self.optimizer.step()\n", - " if batch_idx % log_interval == 0:\n", - " print('Train Epoch: 1 [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n", - " batch_idx * len(data), len(self.train_loader.dataset),\n", - " 100. * batch_idx / len(self.train_loader), loss.item()))\n", - " self.loss = loss.item()\n", - " torch.save(self.model.state_dict(), 'model.pth')\n", - " torch.save(self.optimizer.state_dict(), 'optimizer.pth')\n", + " self.optimizer.zero_grad()\n", + " output = self.model(data)\n", + " loss = F.nll_loss(output, target)\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " if batch_idx % log_interval == 0:\n", + " print('Train Epoch: 1 [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n", + " batch_idx * len(data), len(self.train_loader.dataset),\n", + " 100. * batch_idx / len(self.train_loader), loss.item()))\n", + " self.loss = loss.item()\n", + " torch.save(self.model.state_dict(), 'model.pth')\n", + " torch.save(self.optimizer.state_dict(), 'optimizer.pth')\n", " self.training_completed = True\n", " self.next(self.local_model_validation)\n", "\n", " @collaborator\n", " def local_model_validation(self):\n", - " self.local_validation_score = inference(self.model,self.test_loader)\n", - " print(f'Doing local model validation for collaborator {self.input}: {self.local_validation_score}')\n", + " self.local_validation_score = inference(self.model, self.test_loader)\n", + " print(\n", + " f'Doing local model validation for collaborator {self.input}: {self.local_validation_score}')\n", " self.next(self.join, exclude=['training_completed'])\n", "\n", " @aggregator\n", - " def join(self,inputs):\n", - " self.average_loss = sum(input.loss for input in inputs)/len(inputs)\n", - " self.aggregated_model_accuracy = sum(input.agg_validation_score for input in inputs)/len(inputs)\n", - " self.local_model_accuracy = sum(input.local_validation_score for input in inputs)/len(inputs)\n", + " def join(self, inputs):\n", + " self.average_loss = sum(input.loss for input in inputs) / len(inputs)\n", + " self.aggregated_model_accuracy = sum(\n", + " input.agg_validation_score for input in inputs) / len(inputs)\n", + " self.local_model_accuracy = sum(\n", + " input.local_validation_score for input in inputs) / len(inputs)\n", " print(f'Average aggregated model validation values = {self.aggregated_model_accuracy}')\n", " print(f'Average training loss = {self.average_loss}')\n", " print(f'Average local model validation values = {self.local_model_accuracy}')\n", @@ -283,23 +306,27 @@ " self.optimizer = [input.optimizer for input in inputs][0]\n", " self.current_round += 1\n", " if self.current_round < self.rounds:\n", - " self.next(self.aggregated_model_validation, foreach='collaborators', exclude=['private'])\n", + " self.next(self.aggregated_model_validation,\n", + " foreach='collaborators', exclude=['private'])\n", " else:\n", " self.next(self.end)\n", - " \n", + "\n", " @aggregator\n", " def end(self):\n", - " print(f'This is the end of the flow') " + " print(f'This is the end of the flow')" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "2aabf61e", "metadata": {}, "source": [ - "You'll notice in the `FederatedFlow` definition above that there were certain attributes that the flow was not initialized with, namely the `train_loader` and `test_loader` for each of the collaborators. These are **private_attributes** that are exposed only throught he runtime. Each participant has it's own set of private attributes: a dictionary where the key is the attribute name, and the value is the object that will be made accessible through that participant's task. \n", + "You'll notice in the `FederatedFlow` definition above that there were certain attributes that the flow was not initialized with, namely the `train_loader` and `test_loader` for each of the collaborators. These are **private attributes** of the participant which are specified via a callback function while instantiating the participant. The callback function returns the private attributes in form of a dictionary where the key is the attribute name, and the value is the object that will be made accessible to that participant's task\n", + "\n", + "The callback function, `callable_to_initialize_collaborator_private_attributes`, segment shards of the MNIST dataset for four collaborators: `Portland`, `Seattle`, `Chandler`, and `Bangalore`. Each collaborator has their own slice of the dataset that is accessible through the `train_loader` and `test_loader` attributes. Parameters required by the callback function `index`, `n_collaborators`, `train_dataset`, `test_dataset` and `batch_size` are passed appropriate values with the same names in the Collaborator constructor\n", "\n", - "Below, we segment shards of the MNIST dataset for **four collaborators**: Portland, Seattle, Chandler, and Portland. Each has their own slice of the dataset that's accessible via the `train_loader` or `test_loader` attribute. Note that the private attributes are flexible, and you can choose to pass in a completely different type of object to any of the collaborators or aggregator (with an arbitrary name). These private attributes will always be filtered out of the current state when transfering from collaborator to aggregator, or vice versa. " + "Note that the private attributes are flexible, and you can choose to pass in a completely different type of object to any of the collaborators or aggregator (with an arbitrary name). These private attributes will always be filtered out of the current state when transfering from collaborator to aggregator, or vice versa" ] }, { @@ -309,30 +336,46 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup participants\n", - "aggregator = Aggregator()\n", - "aggregator.private_attributes = {}\n", + "# Aggregator\n", + "aggregator_ = Aggregator()\n", "\n", - "# Setup collaborators with private attributes\n", - "collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", - "for idx, collaborator in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx::len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx::len(collaborators)]\n", - " local_test.data = mnist_test.data[idx::len(collaborators)]\n", - " local_test.targets = mnist_test.targets[idx::len(collaborators)]\n", - " collaborator.private_attributes = {\n", - " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", - " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", + "collaborator_names = [\"Portland\", \"Seattle\", \"Chandler\", \"Bangalore\"]\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index, n_collaborators, batch_size, train_dataset, test_dataset):\n", + " train = deepcopy(train_dataset)\n", + " test = deepcopy(test_dataset)\n", + " train.data = train_dataset.data[index::n_collaborators]\n", + " train.targets = train_dataset.targets[index::n_collaborators]\n", + " test.data = test_dataset.data[index::n_collaborators]\n", + " test.targets = test_dataset.targets[index::n_collaborators]\n", + "\n", + " return {\n", + " \"train_loader\": torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True),\n", + " \"test_loader\": torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=True),\n", " }\n", "\n", - "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='single_process')\n", + "# Setup collaborators private attributes via callable function\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, \n", + " n_collaborators=len(collaborator_names),\n", + " train_dataset=mnist_train, \n", + " test_dataset=mnist_test, \n", + " batch_size=64\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=aggregator_, collaborators=collaborators,\n", + " backend=\"ray\")\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "278ad46b", "metadata": {}, @@ -343,24 +386,25 @@ { "cell_type": "code", "execution_count": null, - "id": "16937a65", + "id": "a175b4d6", "metadata": {}, "outputs": [], "source": [ "model = None\n", "best_model = None\n", "optimizer = None\n", - "flflow = FederatedFlow(model,optimizer)\n", + "flflow = FederatedFlow(model, optimizer, checkpoint=True)\n", "flflow.runtime = local_runtime\n", "flflow.run()" ] }, { + "attachments": {}, "cell_type": "markdown", - "id": "c32e0844", + "id": "86b3dd2e", "metadata": {}, "source": [ - "Now that the flow has completed, let's get the final model and accuracy:" + "Now that the flow has completed, let's get the final model and accuracy" ] }, { @@ -376,6 +420,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "5dd1558c", "metadata": {}, @@ -392,12 +437,13 @@ "metadata": {}, "outputs": [], "source": [ - "flflow2 = FederatedFlow(model=flflow.model,optimizer=flflow.optimizer,checkpoint=True)\n", + "flflow2 = FederatedFlow(model=flflow.model, optimizer=flflow.optimizer, checkpoint=True)\n", "flflow2.runtime = local_runtime\n", "flflow2.run()" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a61a876d", "metadata": {}, @@ -447,6 +493,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "b55ccb19", "metadata": {}, @@ -475,6 +522,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "e5efa1ff", "metadata": {}, @@ -493,6 +541,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "3292b2e0", "metadata": {}, @@ -531,6 +580,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "eb1866b7", "metadata": {}, @@ -559,6 +609,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ef877a50", "metadata": {}, @@ -587,6 +638,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "9826c45f", "metadata": {}, @@ -605,6 +657,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "dd962ddc", "metadata": {}, @@ -623,6 +676,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "426f2395", "metadata": {}, diff --git a/openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb b/openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb index af8e9513dc..79e9ec7ec0 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_102_Aggregator_Validation.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "14821d97", "metadata": {}, @@ -10,6 +11,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bd059520", "metadata": {}, @@ -18,6 +20,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "fc8e35da", "metadata": {}, @@ -26,6 +29,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "4dbb89b6", "metadata": {}, @@ -42,6 +46,8 @@ "source": [ "!pip install git+https://github.com/intel/openfl.git\n", "!pip install -r requirements_workflow_interface.txt\n", + "!pip install torch\n", + "!pip install torchvision\n", "\n", "# Uncomment this if running in Google Colab\n", "#!pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", @@ -50,6 +56,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7237eac4", "metadata": {}, @@ -133,6 +140,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "cd268911", "metadata": {}, @@ -171,6 +179,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "b2e45614", "metadata": { @@ -279,13 +288,14 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7a133f9f", "metadata": {}, "source": [ - "You'll notice in the `FederatedFlow` definition above that there were certain attributes that the flow was not initialized with, namely the `train_loader` and `test_loader` for each of the collaborators. These are **private_attributes** that are exposed only throught he runtime. Each participant has it's own set of private attributes: a dictionary where the key is the attribute name, and the value is the object that will be made accessible through that participant's task. \n", + "You'll notice in the `FederatedFlow` definition above that there were certain attributes that the flow was not initialized with, namely the `train_loader` and `test_loader` for each of the collaborators. Each participant has it's own set of private attributes which can be set using callback function while instantiating the participant. The callback function returns the private attributes (`train_loader` & `test_loader`) in form of a dictionary where the key is the attribute name, and the value is the object that will be made accessible to that participant's task\n", "\n", - "Below, we segment shards of the MNIST dataset for **four collaborators**: Portland, Seattle, Chandler, and Portland. Each has their own slice of the dataset that's accessible via the `train_loader` or `test_loader` attribute. Note that the private attributes are flexible, and you can choose to pass in a completely different type of object to any of the collaborators or aggregator (with an arbitrary name). These private attributes will always be filtered out of the current state when transfering from collaborator to aggregator, or vice versa. " + "Callback function, `callable_to_initialize_collaborator_private_attributes`, segment shards of the MNIST dataset for four collaborators: `Portland`, `Seattle`, `Chandler`, and `Bangalore`. Callback function, `callable_to_initialize_aggregator_private_attributes`, returns the private attribute `test_loader` of the Aggregator." ] }, { @@ -295,37 +305,55 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup participants\n", - "aggregator = Aggregator()\n", - "\n", - "# Setup collaborators with private attributes\n", "collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", "\n", - "aggregator_test = deepcopy(mnist_test)\n", - "aggregator_test.targets = mnist_test.targets[len(collaborators)::len(collaborators)+1]\n", - "aggregator_test.data = mnist_test.data[len(collaborators)::len(collaborators)+1]\n", - "aggregator.private_attributes = {\n", - " 'test_loader': torch.utils.data.DataLoader(aggregator_test,batch_size=batch_size_train, shuffle=True)\n", - "}\n", + "def callable_to_initialize_aggregator_private_attributes(n_collaborators, test_dataset, batch_size_train):\n", + " aggregator_test = deepcopy(test_dataset)\n", + " aggregator_test.targets = test_dataset.targets[n_collaborators::n_collaborators+1]\n", + " aggregator_test.data = test_dataset.data[n_collaborators::n_collaborators+1]\n", + " return {\n", + " 'test_loader': torch.utils.data.DataLoader(aggregator_test,batch_size=batch_size_train, shuffle=True)\n", + " }\n", + "\n", + "# Setup Aggregator private attributes via callable function\n", + "aggregator = Aggregator(\n", + " name=\"agg\",\n", + " private_attributes_callable=callable_to_initialize_aggregator_private_attributes,\n", + " n_collaborators=len(collaborator_names),\n", + " test_dataset=mnist_test, batch_size_train=batch_size_train\n", + ")\n", "\n", - "for idx, collaborator in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx::len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx::len(collaborators)]\n", - " local_test.data = mnist_test.data[idx::len(collaborators)+1]\n", - " local_test.targets = mnist_test.targets[idx::len(collaborators)+1]\n", - " collaborator.private_attributes = {\n", - " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", - " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", + "# Setup collaborators private attributes via callable function\n", + "def callable_to_initialize_collaborator_private_attributes(index, n_collaborators, train_dataset, test_dataset, batch_size_train):\n", + " local_train = deepcopy(train_dataset)\n", + " local_test = deepcopy(test_dataset)\n", + " local_train.data = train_dataset.data[index::n_collaborators]\n", + " local_train.targets = train_dataset.targets[index::n_collaborators]\n", + " local_test.data = test_dataset.data[index::n_collaborators]\n", + " local_test.targets = test_dataset.targets[index::n_collaborators]\n", + " \n", + " return {\n", + " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", + " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", " }\n", "\n", - "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='single_process')\n", + "collaborators=[]\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0, num_gpus=0,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, n_collaborators=len(collaborator_names),\n", + " train_dataset=mnist_train, test_dataset=mnist_test, batch_size_train=batch_size_train,\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='ray')\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "0525eaa9", "metadata": {}, @@ -343,12 +371,13 @@ "model = None\n", "best_model = None\n", "optimizer = None\n", - "flflow = AggregatorValidationFlow(model,optimizer)\n", + "flflow = AggregatorValidationFlow(model, optimizer)\n", "flflow.runtime = local_runtime\n", "flflow.run()" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8b9f8d25", "metadata": {}, diff --git a/openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb b/openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb index 00c643542a..11e6f89104 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_103_Cyclic_Institutional_Incremental_Learning.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "14821d97", "metadata": {}, @@ -28,6 +29,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "fc8e35da", "metadata": {}, @@ -36,6 +38,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "4dbb89b6", "metadata": {}, @@ -52,6 +55,8 @@ "source": [ "!pip install git+https://github.com/intel/openfl.git\n", "!pip install -r requirements_workflow_interface.txt\n", + "!pip install torch\n", + "!pip install torchvision\n", "\n", "# Uncomment this if running in Google Colab\n", "#!pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", @@ -60,6 +65,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7237eac4", "metadata": {}, @@ -143,6 +149,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "cd268911", "metadata": {}, @@ -181,6 +188,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "b2e45614", "metadata": { @@ -298,13 +306,24 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7a133f9f", "metadata": {}, "source": [ - "You'll notice in the `CyclicFlow` definition above that each collaborator performs **aggregated_model_validation**, **training**, and **local_model_validation** before passing it's model on to the next collaborator (through the aggregator). \n", + "You'll notice in the `CyclicFlow` definition above that each collaborator performs **aggregated_model_validation**, **training**, and **local_model_validation** before passing it's model on to the next collaborator (through the aggregator)" + ] + }, + { + "cell_type": "markdown", + "id": "a7f4c614", + "metadata": {}, + "source": [ + "Let's define the Participants and runtime now ! Each participant has it's own set of private attributes which can be set using callback function while instantiating the participant. The callback function returns the private attributes in form of a dictionary where the key is the attribute name, and the value is the object that will be made accessible to that participant's task\n", "\n", - "Below, we segment shards of the MNIST dataset for **four collaborators**: Portland, Seattle, Chandler, and Portland **equally and IID**. Each has their own slice of the dataset that's accessible via the `train_loader` or `test_loader` attribute. Note that the private attributes are flexible, and you can choose to pass in a completely different type of object to any of the collaborators or aggregator (with an arbitrary name). These private attributes will always be filtered out of the current state when transfering from collaborator to aggregator, or vice versa. " + "Callback function, `callable_to_initialize_aggregator_private_attributes`, returns the private attribute `test_loader` of the Aggregator.\n", + "\n", + "Callback function, `callable_to_initialize_collaborator_private_attributes`, segment shards of the MNIST dataset for four collaborators: `Portland`, `Seattle`, `Chandler`, and `Bangalore` and returns the private attribute `train_loader` and `test_loader`" ] }, { @@ -314,37 +333,58 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup participants\n", - "agg = Aggregator()\n", + "collaborator_names = ['Portland', 'Seattle','Chandler','Bangalore']\n", "\n", - "# Setup collaborators with private attributes\n", - "collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", - "\n", - "aggregator_test = deepcopy(mnist_test)\n", - "aggregator_test.targets = mnist_test.targets[len(collaborators)::len(collaborators)+1]\n", - "aggregator_test.data = mnist_test.data[len(collaborators)::len(collaborators)+1]\n", - "aggregator.private_attributes = {\n", - " 'test_loader': torch.utils.data.DataLoader(aggregator_test,batch_size=batch_size_train, shuffle=True)\n", - "}\n", - "\n", - "for idx, col in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx::len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx::len(collaborators)]\n", - " local_test.data = mnist_test.data[idx::len(collaborators)+1]\n", - " local_test.targets = mnist_test.targets[idx::len(collaborators)+1]\n", - " col.private_attributes = {\n", - " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", - " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", + "def callable_to_initialize_aggregator_private_attributes(n_collaborators, test_dataset,\n", + " batch_size):\n", + " aggregator_test = deepcopy(test_dataset)\n", + " aggregator_test.targets = test_dataset.targets[n_collaborators::n_collaborators+1]\n", + " aggregator_test.data = test_dataset.data[n_collaborators::n_collaborators+1]\n", + "\n", + " return {\n", + " 'test_loader': torch.utils.data.DataLoader(aggregator_test, batch_size=batch_size, shuffle=True)\n", + " }\n", + "\n", + "# Setup Aggregator private attributes via callable function\n", + "agg = Aggregator(\n", + " name=\"agg\",\n", + " private_attributes_callable=callable_to_initialize_aggregator_private_attributes,\n", + " n_collaborators=len(collaborator_names), test_dataset=mnist_test,\n", + " batch_size=batch_size_test\n", + ")\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index, n_collaborators,\n", + " batch_size_train, train_dataset, test_dataset):\n", + " local_train = deepcopy(train_dataset)\n", + " local_test = deepcopy(test_dataset)\n", + " local_train.data = train_dataset.data[index::n_collaborators]\n", + " local_train.targets = train_dataset.targets[index::n_collaborators]\n", + " local_test.data = test_dataset.data[index::n_collaborators+1]\n", + " local_test.targets = test_dataset.targets[index::n_collaborators+1]\n", + "\n", + " return {\n", + " 'train_loader': torch.utils.data.DataLoader(local_train, batch_size=batch_size_train, shuffle=True),\n", + " 'test_loader': torch.utils.data.DataLoader(local_test, batch_size=batch_size_train, shuffle=True)\n", " }\n", "\n", - "local_runtime = LocalRuntime(aggregator=agg, collaborators=collaborators, backend='single_process')\n", + "# Setup collaborators private attributes via callable function\n", + "collaborators=[]\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator( \n", + " name=collaborator_name, num_cpus=0, num_gpus=0,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, n_collaborators=len(collaborator_names), batch_size_train=batch_size_train,\n", + " train_dataset=mnist_train, test_dataset=mnist_test\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=agg, collaborators=collaborators, backend='ray') \n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "0525eaa9", "metadata": {}, @@ -368,6 +408,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ad93c508", "metadata": {}, @@ -482,6 +523,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "e58fe9cd", "metadata": {}, @@ -496,45 +538,50 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup participants\n", - "agg = Aggregator()\n", - "\n", - "# Setup collaborators with private attributes\n", "collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", - "\n", - "aggregator_test = deepcopy(mnist_test)\n", - "aggregator_test.targets = mnist_test.targets[len(collaborators)::len(collaborators)+1]\n", - "aggregator_test.data = mnist_test.data[len(collaborators)::len(collaborators)+1]\n", - "aggregator.private_attributes = {\n", - " 'test_loader': torch.utils.data.DataLoader(aggregator_test,batch_size=batch_size_train, shuffle=True)\n", - "}\n", - "\n", - "for idx, col in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx::len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx::len(collaborators)]\n", - " if col.name == 'Portland':\n", + "\n", + "def callable_to_initialize_aggregator_private_attributes(n_collaborators, test_dataset, batch_size):\n", + " aggregator_test = deepcopy(test_dataset)\n", + " aggregator_test.targets = test_dataset.targets[n_collaborators::n_collaborators+1]\n", + " aggregator_test.data = test_dataset.data[n_collaborators::n_collaborators+1]\n", + "\n", + " return {\n", + " 'test_loader': torch.utils.data.DataLoader(aggregator_test, batch_size=batch_size, shuffle=True)\n", + " }\n", + "\n", + "# Setup Aggregator private attributes via callable function\n", + "agg = Aggregator(\n", + " name=\"agg\",\n", + " private_attributes_callable=callable_to_initialize_aggregator_private_attributes,\n", + " n_collaborators = len(collaborator_names), test_dataset=mnist_test,\n", + " batch_size=batch_size_test\n", + ")\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index, collaborator_name, n_collaborators, batch_size_train, train_dataset, test_dataset):\n", + " local_train = deepcopy(train_dataset)\n", + " local_test = deepcopy(test_dataset)\n", + " local_train.data = train_dataset.data[index::n_collaborators]\n", + " local_train.targets = train_dataset.targets[index::n_collaborators]\n", + " if collaborator_name == 'Portland':\n", " # Remove the 0 class from Portland\n", " mask = local_train.targets != 1\n", " local_train.data = local_train.data[mask]\n", " local_train.targets = local_train.targets[mask]\n", - " if col.name == 'Seattle':\n", + " if collaborator_name == 'Seattle':\n", " # Seattle has 500 samples of class 1 (exclusively)\n", " mask = local_train.targets == 1\n", " local_train.data = local_train.data[mask]\n", " local_train.targets = local_train.targets[mask]\n", " local_train.data = local_train.data[:500]\n", " local_train.targets = local_train.targets[:500]\n", - " if col.name == 'Chandler':\n", + " if collaborator_name == 'Chandler':\n", " # Chandler has 300 samples of class 2 (exclusively)\n", " mask = local_train.targets == 2\n", " local_train.data = local_train.data[mask]\n", " local_train.targets = local_train.targets[mask]\n", " local_train.data = local_train.data[:300]\n", " local_train.targets = local_train.targets[:300]\n", - " if col.name == 'Bangalore':\n", + " if collaborator_name == 'Bangalore':\n", " # Bangalore has 300 samples of class 3 (exclusively)\n", " mask = local_train.targets == 3\n", " local_train.data = local_train.data[mask]\n", @@ -542,14 +589,26 @@ " local_train.data = local_train.data[:500]\n", " local_train.targets = local_train.targets[:500]\n", " # Test data is left unchanged (all classes represented)\n", - " local_test.data = mnist_test.data[idx::len(collaborators)+1]\n", - " local_test.targets = mnist_test.targets[idx::len(collaborators)+1]\n", - " col.private_attributes = {\n", - " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", - " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", + " local_test.data = test_dataset.data[index::n_collaborators+1]\n", + " local_test.targets = test_dataset.targets[index::n_collaborators+1]\n", + " return {\n", + " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", + " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", " }\n", "\n", - "local_runtime = LocalRuntime(aggregator=agg, collaborators=collaborators, backend='single_process')\n", + "# Setup collaborators private attributes via callable function\n", + "collaborators=[]\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0, num_gpus=0,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, collaborator_name=collaborator_name, n_collaborators=len(collaborator_names),\n", + " batch_size_train=batch_size_train, train_dataset=mnist_train, test_dataset=mnist_test,\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=agg, collaborators=collaborators, backend='ray')\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')" ] }, @@ -563,7 +622,7 @@ "model = None\n", "best_model = None\n", "optimizer = None\n", - "clflow2 = CyclicLearningFlow(model,optimizer,rounds=4)\n", + "clflow2 = CyclicLearningFlow(model, optimizer, rounds=4)\n", "clflow2.runtime = local_runtime\n", "clflow2.run()" ] @@ -578,12 +637,13 @@ "model = None\n", "best_model = None\n", "optimizer = None\n", - "flflow2 = FederatedFlow(model,optimizer,rounds=4)\n", + "flflow2 = FederatedFlow(model, optimizer, rounds=4)\n", "flflow2.runtime = local_runtime\n", "flflow2.run()" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "285d63a9", "metadata": {}, @@ -592,6 +652,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8b9f8d25", "metadata": {}, diff --git a/openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb b/openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb new file mode 100644 index 0000000000..5046f373ca --- /dev/null +++ b/openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb @@ -0,0 +1,363 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow Interface 104: Working with Keras\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_104_Keras_MNIST_with_GPU.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, we will demonstrate how to use Keras with Workflow Interface. Even though this tutorial is meant to be ran with GPU, in case GPU is not available this can be ran on CPU as well by changing `num_gpus=0.3` to `num_gpus=0`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting Started" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we start by installing the necessary dependencies for the workflow interface. Please note if you intent to run this tutorial on GPU then install CUDA and cuDNN versions for TensorFlow 2.7 as mentioned [here](https://www.tensorflow.org/install/source#gpu)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/intel/openfl.git\n", + "!pip install -r requirements_workflow_interface.txt\n", + "!pip install tensorflow==2.7.0\n", + "\n", + "# Uncomment this if running in Google Colab\n", + "# !pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", + "# import os\n", + "# os.environ[\"USERNAME\"] = \"colab\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Configure Tensorflow to allocate GPU memory as it is needed by the processes (instead of TF default policy to allocate nearly all of the memory on GPUs). Refer [Limiting GPU memory growth](https://www.tensorflow.org/guide/gpu#limiting_gpu_memory_growth)\n", + "\n", + "IMPORTANT NOTE: This is needed to demonstrate fractional usage of GPUs by Ray backend and avoid conflict between Ray and Tensorflow while allocating GPU memory" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "os.environ[\"TF_FORCE_GPU_ALLOW_GROWTH\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We begin with the quintessential example of a small keras CNN model trained on the MNIST dataset. Let's start define our dataloaders, model, optimizer, and some helper functions like we would for any other deep learning experiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from tensorflow.keras.layers import Flatten, Dense, Dropout, Conv2D, MaxPool2D\n", + "from tensorflow.keras.models import Sequential\n", + "from tensorflow.keras.datasets import mnist\n", + "from tensorflow.keras.utils import to_categorical\n", + "\n", + "nb_classes = 10\n", + "batch_size=32\n", + "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n", + "print(\"X_train original shape\", X_train.shape)\n", + "print(\"y_train original shape\", y_train.shape)\n", + "\n", + "X_train = X_train.astype(\"float32\")\n", + "X_test = X_test.astype(\"float32\")\n", + "X_train /= 255.0\n", + "X_test /= 255.0\n", + "print(\"Training matrix shape\", X_train.shape)\n", + "print(\"Testing matrix shape\", X_test.shape)\n", + "\n", + "Y_train = to_categorical(y_train, nb_classes)\n", + "Y_test = to_categorical(y_test, nb_classes)\n", + "\n", + "train_dataset=(X_train, Y_train)\n", + "test_dataset=(X_test, Y_test)\n", + "\n", + "model = Sequential([\n", + " Conv2D(filters=32, kernel_size=(3, 3), activation=\"relu\", input_shape=(28, 28, 1)),\n", + " MaxPool2D(),\n", + " Flatten(),\n", + " Dense(512, activation=\"relu\"),\n", + " Dropout(0.2),\n", + " Dense(512, activation=\"relu\"),\n", + " Dropout(0.2),\n", + " Dense(nb_classes, activation=\"softmax\"),\n", + "])\n", + "\n", + "model.compile(optimizer=\"SGD\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n", + "print(model.summary())\n", + "\n", + "\n", + "def inference(model, test_loader, batch_size):\n", + " x_test, y_test = test_loader\n", + " loss, accuracy = model.evaluate(\n", + " x_test,\n", + " y_test,\n", + " batch_size=batch_size,\n", + " verbose=0\n", + " )\n", + " accuracy_percentage = accuracy * 100\n", + " print(f\"Test set: Avg. loss: {loss}, Accuracy: {accuracy_percentage:.2f}%\")\n", + " return accuracy" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we import the `FLSpec`, `LocalRuntime`, and placement decorators.\n", + "\n", + "- `FLSpec` – Defines the flow specification. User defined flows are subclasses of this.\n", + "- `Runtime` – Defines where the flow runs, infrastructure for task transitions (how information gets sent). The `LocalRuntime` runs the flow on a single node.\n", + "- `aggregator/collaborator` - placement decorators that define where the task will be assigned" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openfl.experimental.interface import FLSpec\n", + "from openfl.experimental.runtime import LocalRuntime\n", + "from openfl.experimental.placement import aggregator, collaborator\n", + "import numpy as np\n", + "\n", + "\n", + "def FedAvg(models):\n", + " new_model = models[0]\n", + " state_dicts = [model.weights for model in models]\n", + " state_dict = new_model.weights\n", + " for idx, _ in enumerate(models[1].weights):\n", + " state_dict[idx] = np.sum(np.array([state[idx]\n", + " for state in state_dicts], dtype=object), axis=0) / len(models)\n", + " new_model.set_weights(state_dict)\n", + " return new_model" + ] + }, + { + "attachments": { + "image.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4sAAAK/CAYAAADAj1hsAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAEnQAABJ0Ad5mH3gAAP+lSURBVHhe7J0FYFzXsUDHttgW2jIzO8bEDsdhpjZNGdNfpl9mhl/GtCmklFKaNmnTBpqkYQcccBI7jplJtiVZspjlP2d2r/28khyDtJKsOc7Lrnbf7j64c+/QndtvnyKO4ziO4ziO4ziOE6F//NFxHMdxHMdxHMdx9uPGouM4juM4juM4jtMGNxYdx3Ecx3Ecx3GcNrix6DiO4ziO4ziO47TBjUXHcRzHcRzHcRynDW4sOo7jOI7jOI7jOG1wY9FxHMdxHMdxHMdpgxuLjuM4juM4juM4ThvcWHQcx3Ecx3Ecx3Ha4Mai4ziO4ziO4ziO0wY3Fh3HcRzHcRzHcZw2uLHoOI7jOI7jOI7jtMGNRcdxHMdxHMdxHKcNbiw6juM4juM4juM4bXBj0XEcx3Ecx3Ecx2mDG4uO4ziO4ziO4zhOG9xYdBzHcRzHcRzHcdrgxqLjOI7jOI7jOI7TBjcWHcdxHMdxHMdxnDa4seg4juM4juM4juO0wY1Fx3Ecx3Ecx3Ecpw1uLDqO4ziO4ziO4zhtcGPRcRzHcRzHcRzHaYMbi47jOI7jOI7jOE4b3Fh0HMdxHMdxHMdx2uDGouM4juM4juM4jtMGNxYdx3Ecx3Ecx3GcNrix6DiO4ziO4ziO47TBjUXHcRzHcRzHcRynDW4sOo7jOI7jOI7jOG1wY9FxHMdxHMdxHMdpgxuLjuM4juM4juM4ThvcWHQcx3Ecx3Ecx3Ha4Mai4ziO4ziO4ziO0wY3Fh3HcRzHcRzHcZw2uLHoOI7jOI7jOI7jtMGNRcdxHMdxHMdxHKcNbiw6juM4juM4juM4bei3T4k/dxzHcdqhVXvJZv1fi25NLfqo3WboOb0LdZzOpV8/tn6S0r+f6H/2yDaAPxzHcZyk4sai4zjOy1BS3STLd9bISzur5ektVbK5rF6qG1vUeBSp0UfHcTqPtAH9ZMigVJk1YpAM08d5owbJSaMHyfiCjPgejuM4TrJwY9FxHKcDNu2pl4fW75XHNlTIutJa2VvbLCU1TVLT0CJNrbHoIhFHx3E6j/79+klGaj/Jy0yVjJT+UpCVYoYiRuNZE3JkwdhsGZg2IL634ziO05W4seg4jhOBVNPyumZ5bGOFLN5cKY/r46rdaijqa47jdA8Yh6Py0mTeyEFyxoRcuWhavkwtzLT0VMdxHKfrcGPRcRwnTkNzq2zf2yBPbKqU3z69U17YXi1VDZ5m6jg9BUzD0Xnp8raTh8s1c4bI9KGZkuVRRsdxnC7DjUXHcRyFjnBjaZ38bWmJ/OyxHbKrstFecxyn55GZ2t+MxU+eN0bmjBxoqauO4zhO5+NLZziO4yi1jS1y67IS+f3Tu6S4uskNRcfpwdQ3t8q9q8rl6//dIjsrG61KseM4jtP5uLHoOI6j3Las1JTPbXsbbN6i4zg9F3KimEe8ZFuV/PSxItlT0xR/x3Ecx+lM3Fh0HKdP09jcKutL6uTOl/bY8hjMW3Qcp+fTqhZjcVWT3P5iqRmNFKZyHMdxOhc3Fh3H6dNUNrTI3av2yHPbq6Ss1qMTjtObIB11w546+ddLpbJJHx3HcZzOxY1Fx3H6LEQmWHD/j88W2zxFx3F6H6SN/+2FEnlpZ62ve+o4jtPJuLHoOE6fZVdVkzyxqULWFNdKXZMvkeE4vRUKVD27rUoNxpr4K47jOE5n4Mai4zh9FgpkrCups3mKvoiQ4/ReCChuKK2TLeX18Vccx3GczsCNRcdx+iyV9c2mXLqd6Di9n81l9bK1vCH+l+M4jtMZuLHoOE6fpbK+RbaUNcg+Dys6Tq9nV1WjzUF2HMdxOg83Fh3H6bNU1TfLViKLbis6Tq+nvqnVqqM6juM4nYcbi47j9FmaWvdJTWOrp6E6znEAhqKvk+o4jtO5uLHoOE6fpalFjUWvguo4xwVkCHiWgOM4TufixqLjOH0WKig2q8HoOI7jOI7jtKXfPq/s4DhOH+UPz+6W625eHf/LSSa5mSkyKjdNstNTJCOlv6T07yf9+hEZ2mcLqzcS9W1skeKqJitccigKB6VKvn5fS6vIhj118Ve7j5yMATJ0UJqk63lxPMylc5LDhxaOkutfNTn+l+M4jnOsuLHoOE6fxY3F5KM2oRlRZ03IletOGS5zRg6S0Wo0DlIDC4OxRQ3FivpmKa1pltW7a+WOl0rlb0tLpLaxVVrbGa4yU/vL1bMGy8KJubZu5hf/s7lb56BybqeOy5ZrZg+RIQNT5Yv3bLYlHZzk4Mai4zhO5+JpqI7jOE7SGKwG1KfOHyM/vmayXKVG3tTCDMnJHCADsCIVHok6ji/IkAum5skXLh4nv3j1FBmZkyZpA2L7RPnMBWNtO29yvhqOA+Kvdh9vmj9Uvn7ZeHnVnEIZmD7AjGPHcRzH6a24seg4juMkjXedNsIMqYmDM6SxuVUeXr9XfvDwdnnv39fK6/6wUt76l9XytXu3yKINe20dzOHZaXL+lHz55hUTZFReevxbDjClMFPG6OsD0/tbGmt3Myo3Xc8t01JR++sB8c9xHMdxeituLDqO4zhdDvMSpw/NkktnFMhUNfCYh/jPF0vll0/ulJue3SV/fb5Y/r60xFJO/7Rkt/ziiSL5x7JSWV9aJ8OyUy3V9NxJeTI8Jy3+jTHCfMeeAtHPtAE+tDqO4zjHBz6iOY7jOF1OdsYAuXh6vozLT5cUNaiWFVXLjYt3yr+Xl8rKXbVS1RBbwoRo46ayejMkf/v0Tnlk/V6hYC2pqedPybP0VMdxHMdxkoMbi47jOE6Xk50+wCKDVD+tUcNwfUm9PL+96pDFaJbuqJZHN1TIFjUeKV5DFJHIHfMaKWyTpwZkqv7dr18/S/mkuAyvkQLK61HCZziO3IwU2y9s/M3rWWn9251jyOf4TrastAGSwffocwxYthz9/CC+V5/zHumwbKn6Zbafvs/rjuM4jtPb8GqojuP0WbwaavJgXuFnLxwr184tlCEDU+Tm54rl4//eKMXVh14WAyOOqqIYe7WNLVKuRuNo/a5XzRki7zxthIzISTNjjjUzq9UILa1pkp2VjfKdB7fJ3Sv3xL9F5IRhWXKNfuaM8bkyWH+fQjsBoprb9zbI2uJa+fGiHfY8yuf0uC+eXmDPMWA3l9XJZfo3xwEcE785bWiWjM5NN6MSY7GuqdWOZ5e+99fnS+Qni7bb/k7X4dVQHcdxOhd3dTqO4zhdTrUaeo9vrJCq+maLAp46Lke+cPFYee2JhVKQlRLfqy0YcqSlMnexSI0uDDDgOwamqVGm/3B5shEUHKSvDUzrvz+yiFH49lOGy1cuHS9vPXm4nDY+R6YUZtl+bKyHyFxKlt54tRqy37hsgpyu+0TBWB2fny4z1eA8W/ejSA9LfvD60Ow0GV+QafMms9RoxajFA8uGAZuZwvEMkPSU2PE4juM4Tm9iwFeU+HPHcZw+xbKiGvnX8tL4X05XguFUUdcsM0cMlGFqoA3LSZMJBRkyNi9dxuRnyIxhA81oG5uXIdkZKVJR3yJNTFZsB8wujMbtFQ0W3SOVlAX8uZ8Ux3lqc5U8t71KKtUwnTIkUz51wRg5e1KepYU+t61abn+xVBap4frUlkp5dluV7KlpsuMZq8fBnMit5Q2yYU+9RSrhkukFenxZatSmSqYaoqSi3rasVO5fWy4vbK+Wl3bWykPr9trcSwzVYWpAcuQYuLcuK5GH9b1n9Xf5XqdrwQlx2YxYFNhxHMc5dtxYdBynz+LGYvJgsX3mHaq9ZsZgwcBUi8yx9MXZk3LljAk5ctLoQXLC8CxLWSVSh3HGPEDmJGIMBjAUMbye3FQpF0/LVyMvXSrVsGMZji/fs9kMQNI/h6hRSiTxHaeOsMjeog0VVlTnl08WyRP6WTabE6nflZeZKnNHDlRjcIDsrGyQTWosbounowZjccigVGlUA5bf/cp9W+SOl/bI0xicW6vsNzE+qfQ6d+Qg+xzt69sPbJP71pS7oZgk3Fh0HMfpXDwN1XEcx0kaLI/x40e3yz+XlagBFYveNTS3WoEY0kMvmJovHzhrlNz29pny97edYAvcX3FCgb1P6umRwEL+80YNkh0VDTYP8ebni+XfL7V1DixRQ++Gx3eoEbrP0lkxMgt1aw+ikCz1wSNguJbVxp47juM4zvGGG4uO4zhOUsE4+/p/t8grf7tC3vW3tVbshtd2VjSq4Xgg9XRodqpcqMbjFy4eJze9cZpMGHxky2as3F0r33lwq1zx6+Vy0S9elLtWHCh4k0hz6z4pqW6UppbWWJrsoAMFcKJU1rfIf9eU26PjOI7jHO+4seg4juMkFaJxxdVNsqa4Vh5cVy4/fWyHfO7uTfK+29bKe29da1VD71tdZtFA1mQkLZVlN95/5sj9KZ6HAxHLEv2dzWX1ViSHYjks8H/WxFy5dEaBVVP95Hlj5JtXTJAvXDTW0kz5PbYBCUtvQG0jx91o0dBWLyTuOI7j9AHcWHQcx3G6hVg0r0le2FFtRWDuXLFH/vZCsfz+6V3yyyd3yq90u39NuVTVt1hV06tmDrY5gUcCVVFZzuLqWYPl/WeNlI+cM9qMzveeMULecdpwue6UYfKm+cPk4mkFto5jfyuf0z5Nra1mMDqO4zhOX8GNRcdxHKfLYc4hEb3zJufJrBEDJb+D5TKIOobCQz9/vEj+vKRYVhXXWnXRcfkZVvn0cGH9xYmDM+WVs4fIx84dLd+4fIJ84rwxZnSeOSHXqrFS2ZRo4fKdNR4xdBzHcZwE3Fh0HMdxuhwMs9vfPlP++7458pVLxsmCMdnxdzqmsqFZVuyqkSc2VdjChWkp/Wwdw8OFZTVePXeIfOeqiXKWGocYj7urYobhA2v3yt0ry+S3T++UT/x7g3zgtnWyp7bJop2O4ziO48RwY9FxHMfpcqhkmk6ap9p649RwxHjsaqYNzZJrZg+RDDUSWbbj+w9tk1f8ZoW8/o+r5OP/Wi+fv3uT/OKJnbb0BfMUWaCfJTscx3Ecx4nhxqLjOI7T5TS2tMq2igZpbN5nC9+zHt6ckQPj73bMyNy0WBRSbTiWq6htOvwqpFlqJDLXEfuPNRNfLKqR5btqbMmOospG2VXVaN/JPkQeWVC/vxuLjuM4jrMfNxYdx3GcLqe8tln+smS3pYHmZqTImRNy5D1njJQ3zR9qC/KzQD8M6NdPcvR9Fui/ds4QiwzOHjHQys5QCGenGnlRSBul5AyRy8SoIAmlYQoiUU2K3STagtOHZskr9DeY10iK67GaimSxtuiPsiRkKt93rF/oOI7jON2IG4uO4zhOl1Na0yS/fXqXLN5cac8nDcmU/zl1uHz+onHyntNHWrVSit+cPzVPLp9RINedMlw+df5Yed2JQyU7fYAtrM8cw/WldfFvjMFyGI3NrTYfcWxeukUsT1RDk2ghayESUWxRC47lN86cmCsXTMm3fdgW6t9vOGmovHXBMP0724rrHOuUxdrGFvseDEWW6Thx1CCZr8fD/EnHcRzH6W0M+IoSf+44jtOnCFU3na4HI6xGDamyumbJSU8xQ2qQPhYOSrW1E4nuvU0NxLeo4fbquYVyytgcGZGTpp/sJ1vLG+SfL5bI75/ZJVv0eRSijhieY/PTrfIpi/ifMynX9qOKKvBdVD09VR+phHrp9HyLWr7jtBG6b57kZqbInppmM2Jz1DAt12N8fnu1PLO1yj5/yfQCmTEsy+Y+btpTL39bWmKvtwfHMEWPZ8LgDBmpBuIZ43PkHDWCOf+ntlTG93K6CpwAl80oiP/lOI7jHCtuLDqO02dxYzH5sEzF4xsrZMWuWkmj4E3/WIILKaTRNFLmE64trpPblpXKN+/fKnev3GPGXGLkb+XuWhmkhuCo3AxLb83LSrXlNR7dUGFVVHdUxOYm8h4GI/MS8zJTzUDcWxszCn/91E752eNFMlQN14lq5LHv+pI6eWBtuf3GkRiLW8oabF7lJDUa+Y1sNYjz9XFdaZ38d03s+5yuw41Fx3GczqXfPiX+3HEcp0/xh2d3y3U3r47/5SQT5igSEWS9RRbDTxtw8LIYRCFrGltlZ0WDbCqrt9TOjpg2NNMieqSr8h2knS7ZVi0b99TZ9w7PSbP3MSLTmbcY/x1+g7mURC5LahrVIBxoqawYrev1sy+oIQmktZLGmjogtvQGxu6hoCjPzOED7feYg8kwu760Xp7bHotUOl3HhxaOkutfNTn+l+M4jnOsuLHoOE6fxY1Fxzm+cGPRcRync/ECN47jOI7jOI7jOE4b3Fh0HMdxHMdxHMdx2uDGouM4juM4juM4jtMGNxYdx3Ecx3Ecx3GcNrix6DiO4ziO4ziO47TBjUXHcRzHcRzHcRynDW4sOo7jOI7jOI7jOG1wY9FxHMdxHMdxHMdpgxuLjuM4juM4juM4ThvcWHQcx3Ecx3Ecx3Ha4Mai4ziO4ziO4ziO0wY3Fh3HcRzHcRzHcZw2uLHoOI7jOI7jOI7jtMGNRcdxHMdxHMdxHKcNbiw6juM4juM4juM4bXBj0XEcx3Ecx3Ecx2mDG4uO4ziO4ziO4zhOG9xYdBzHcRzHcRzHcdrgxqLjOI7jOI7jOI7TBjcWHcdxHMdxHMdxnDa4seg4juM4juM4juO0wY1Fx3Ecx3Ecx3Ecpw1uLDqO4ziO4ziO4zhtcGPRcRzH6RSGZafJ604slN+9YZp87+qJMiYvPf7O8UteZoqcOzlP/vim6XLOpDzJ1b8dx3Ec53jBjUXHcRynU5g5PEsunzFYXjlriLzt5OFyxoRcGTIwNf7u8Ulman+ZPCRT3rxgmD5m2N+O4ziOc7zQb58Sf+44jtOn+MOzu+W6m1fH/3KOlY+eM1peNWeI5GakyOTCTPnLc7vl14t3yjNbq+J7xBibny7Dc9IkbUDMsKqoa5b0lP5S29QiW8sapLqxRYYMSpWRuk+OfhejVHPrPmlobtXn+2RPbbOU1jTJhIIMSR3QTzLUQOO7eH9HRYOUVDfpZ/mNVH0/9hvNLftk4546Kdb3Anx+sBqzfL5Fv79Wf5fjqNFH9uN7YN6oQTIofYA9D3DMHMPI3HR57dxC+eQFY+Q7D26Ve1eVy8pdNbK3vlkK9RxG6ft8JzS1tNp37qhotGMdmp0m4/VaNDTvk6y02D67KhtlU1m9PXeOnA8tHCXXv2py/C/HcRznWHFj0XGcPosbi51HSv9+8p2rJsoZ43NkhRpLZ07IlRo1/r55/1b554ul8b1Ejbp+8qVLxsvbTxku+VmxlM1n1ZgcV5Auy4tq5Mv3bpZlO2rkDfOHynvPGCEnj82RRjWsMOR2VjVKfVOr/OulPXLrCyXyg1dOkiEDU2Ti4Ewp0O/atrdBfvzodvm3vv++M0bKdZHf4POfvnOj/HHJbjM8Od7vXT1Jrpo5WA2+NHt/bUmdjM1Llxf1OGgbf19aLGlq6D3+oXkye8RAUXtT+vUTSdXPPrm5Ut8vkdFqDH7s3NFmtDbpDg+tK5ffPb1LHt9UIW88aah88KxRZhhDmRq5/1peKt97aJtsKa+Xt548XP7v8vGyXQ3cyUOypFWH4988tVM+e9cm2985ctxYdBzH6Vw8X8ZxHMc5Zt6yYJicOGqQvLizRm54osiMqWGD0ixyF+WLl4yTy2cUyH1rymTBD56zjYjb4KzYfpmpAyyS9+7TR1g08PN3b5Krf/OS3Lu6zFJaCxK+jxTQ57ZXyftvWyeX37hc/vp8sXz2grGycFKu/OX53ft/AwMS4+xj54y2uZWfOn+MHccDa8vlVb9bIW/+c8xpMCg9RQ1EtQiVKYVZ8uD75khVQ4t88B/r7Xuu+e0KWbShQqYWZsoINQIXbdgr3/jvFsHr+nV9/Px/NsuGPTFD8OPnjtHzLJfX/mGlvO6PK+XulXtsTufr1IjEwAUiokQf/+evq2Xh9UvV2N1hrzuO4zhOT8CNRcdxHOeYOWVcthlZm8vqZU1xrfzyySKprG+2SOPpugVOGZsjZXVN8uj6vbJyd61tj6jBRXQNKBhz7dwhkq+PD+k+/1hWYmmsNzxeJJvUCCPKGKVVrbSn1DD9z6oy2VbeYAbl/DGDLFr5x2d37/+NX6gBu6e2yeZRXjwtXy6ami8VenyL9bNEAZ/cVClfu2+LFFc3Sj/9Bzy/ftEO+eYDW83Q43uWqzH89NZKMw6JMGJIFlU22v5Feg6kweZlDrBzJp32DjVSMS4fXV9hz4kunj8lT43FDPtMgxrKW8oaZHVxnW61srsq9l2O4ziO0xNwY9FxHMc5aki/JI1z2tAs2alG00tqTNU2tpqxtkWNt+nDstR4y96/39BBqWYcrVLDK4AxuH1vzFhk7t4C3b+xZZ/tw/w95hCyD2mmdU0HG4vVaqxt1d9hrt/A9AFy1sRcixyOzkuXC9Qo+/DZo2w7c0KORTpH6THw/UT2iioa9bP1UlXfYobjPWpwMqeQ+YtQrobd35aWyP1rymVcQYa86/QRct0pw2RqYZZkxOchtkdeZqqMykmz41pbUit765pt43yYz0iqa4i4MneRuZQ8Oo7jOE5Pw41Fx3Ec56gZlDZAFk7MMwOIuYAD9W8MNjbm4A1VA226GpLM7ZukBhrFXjCcynULFFc1mUEI/fv1s2IyjS2tloYapV4NKn4jCnMYw2tZqf0thZXf4PHVcwsP2jgeooMYnBil1Q3NbYy0uqYWaQrfp+eCoXuGGpr/c+pwm3941cwh9t2HqnpKxJH0UqKOrZGv55wwIAfoOTJnEjBMMUqDgeo4juM4PQk3Fh3HcZyjhgIyFHIhfZRKqL9/4zS5/31zbLtgar5VPiUt9OLp+VbVNBAzlWJgN4XUT5F9luJpRHc6DPgcv0G08VsPbJWzf7ZUFv704O2tf1ktNz9XbPv2U6PtwO8GYq9QyGbS4Ay55a0z5L73zJFLphfIP14skat/u1x+smj7QVVVE7Hj0K29w+dcec9xHMdxegNuLDqO4zhHDctOzBk50Ob8vfOWtbLgB88ftP15yW4rXkM0bumOaosgDs9O218hFMblZ5ixCS2tsfTPzJTYchhRiGImvhaFaqTb9tbb3EnWPBybF5sXmEiLWpSV9S32m5nxJSsC2en6G/p55j5SAZWI6Xcf2iZv+OMq+eUTRfG9Dg0RxPrmFqvQGj1civfMGD7Qooik2TqO4zhOT8eNRcdxHOeooBIpC/AzR5Co232ry2zZjOhG2idG3LTCrNg6iuUNMqUwU04Zmx3/FpGLpuXLJDXugLmDd67YY+sjUgjmtHE5ZnS9ef4wmTEsSwYmGHdRWPvwrpVlNify3Ml5cumM/Pg7Iqfq93zh4nHyRd0wbjFcKTKDQcjxU9n0S5eMk1FqHBJbJE2UNFSWzsCso9gN0UQMX46FY+oI5k9SsIZ5nBwH1U6ZQ3nu5FyZpcbiC/rbYY6m4ziO4/RkBnxFiT93HMfpUywrqrF175yjY/7obFsyIzsjRX70yHZbMiKR/MxUmaBG2VQ1nDbq+3tqmswgo8AMhhdGHPMbR+dlWAGbB9butSI5kwszZMqQLHucpI9XzSyQYTnpFrVjeY6Vu2otNZSqqVQz5TPMXWShfOYEYqjxeSKE/MbVswZblLC0plkN2BpZX1pvhhsL51O85qQxg+SyGfob2alW+GZpUbVsLq+XuWpY5mQMMINv5rCB+vcgm+84Vj/D4vpr1SisbWq1ZTiIFg7oJzZXcXdVk4zXfcbkp9vi/1SLpUIq7/3l+WIzVjG2F+q5cywsNUK00zk2uNfcR8dxHKdzcGPRcZw+C8sg3LFiz0Fz6ZzDh/RRjLKXdtXI/WvLrXBNInWNrTKgfz8Zmp1mKaR/fm63NDbv089lyoKx2TJzxEDZpYZVjhqcROSIKq4vrTOjCkOOaOKEgkxpUEOMAjUN+lmilUTnKJjD8hQsf4EhGuA9itxEf6NADVO+m3UYWfSf76CQzlg9h9lqEI5SQ5RoIBFAlsJYvKVSnt1abWmxRCA5T76H7/3mA9sslZYiPVQ4peLpeD1Gqr1yrhh/j22sMMN1ln7mhOFZdq2qG1rlt8/skvtXl0uJvjcyJ00K9Xu4frEUXf0x55hwY9FxHKdz6bdPiT93HMfpU2A4vOfva80wcZIDaZ0UfiEKGCqAvuGkofLBs0aaAfduvR+kgFIVNboPhWFuu26mLTnxtxeK5eeHOX+wPSheE+Y+ht/gNX73qY+cZOtE3rh4pzyyfq/t4/QePrRwlFz/qsnxvxzHcZxjxecsOo7jOEnjD2+YJje/ZYa8dl5h/BWx9EzWSGTBetI9v3/1JHn4A3Pl2jlD7P2wJAYRPqJ1LPx/LBCR/OfbZ9rGAv3AvMVPnjfG0lBZI5LNcRzHcfo6Hll0HKfPcvuLpfKRf623uXLeEyaHr142Xi6dVmBFY1YXxxbmZx3Gh9aVy63LSmR9SZ0tmv+NKybYGorcG9YtpKBMKH7z0Lq9srOy0T57NFAF9bzJefKVS8db6iwL/1M4hzmMNz2zS25fXipr9Tic3gVR5w+cNVK+qvfVcRzH6Rx8zqLjOH0W5qa9WFRjFTrdVkwOlWrwkfZb09RiC+KzrdhJoaE98vz2aisSQ+SQVFXmKbLMBQVk9tQ2y10ryuTxjRXHZCgCi/tjqDJnsS5+DBwTBXp+9eROMx6d3gdFk6g+G6206ziO4xwbHll0HKfPgqH468U75ZdPFtncNcdxei9Ux33PGSPkmtmx9GXHcRzn2PE5i47j9FmYn7Zg7CArpuI4Tu8FCZ4+NNOWI3Ecx3E6DzcWHcfpszDHiXXzWPIgVMd0HKd3gaEYW2ZloIzJS4+96DiO43QKrh05jtNnYakEqmBeO3eIDBmYEn/VcZzeRL9+/eTksdkytTBTstMHxF91HMdxOgM3Fh3H6dNQGfO184bKjOEDreCJ4zi9BzICiCZePmOwTBqSIQNYkNNxHMfpNNxYdBynT5OZ2t+qJ146Pd/W3/P5i47TO8AuLBiYIhdNy5cLdRuR4ymojuM4nY0bi47jOMp7zxgpF07Ns/X2HMfp+aSn9LfU0w8tHCVj89IldYA7ehzHcTob14ocx3GULDUS33naCPnQ2aPUYPR0VMfpyWSk9pfzp+TLJ88fYwYjhqPjOI7T+fg6i47jOHFYnH1tSZ3ctWKP/PqpnVJU0WivOY7TMyBLfHxBhlw6vUCunDlYTh2XLYOzUuPvOo7jOJ2NG4uO4zgRWlr3yc7KRvnrC8Xy2IYKeWlXjf1d3+RGo+N0FxSuyctIkZkjsuTsSXlyxQkFMnvEQM8CcBzH6WLcWHQcx+mAB9eWy3/XlMuSbVWyrbxBqhtbpEa3OjUcm1q860wKDFH7WtSKbxZp1a2l6cBjc6O+p0Z8WpZIRk7sMcDrTfUi9VX6WKf7NvCiSP8UtTxSRVLS489143H/c4wPn/vWnVC4BuOQSqdpA/pJjhqJgwemyNShWfKq2UPkzIm5Ujgw1ecoOo7jJAE3Fh3HcV6GbXsb5PGNFfLCjmp5fnu1rNpdI6XVaqw4x4QNPjoE7X/ELuSv6GsYhQ3VIjVluu05sFWXiJRvN0Ow/5h50n/qOdJ/5Cxp5TOtrdKqhuK+3WtENjwpUrxWpGyLfp8anRm5IjkjRPJHiWQViAwcrBuP8efp2dIPg7Ffv5jJaI/6jz8Oes3pKph/yJI2w3PSrMLpuZNyZaFu09VY9OVtHMdxkosbi47jOC8DUcQQUaxtapGGpn3S3Opd57HQ1NQkVVVVUlJSoltx/LFUdu/eLVu2bJaamlopK9sje/fulfra2lg0sVWNvegjhuS+Vjn7govl1W98m5x8xkJZuXKlLFnynDz+2CJZt+olqa9UI5OoYgtRSL1nGIJEFtn66XP+tsgir6dIRmaW5OcXyJDCIVI4ZIgMGVIYf14ohYXh+RDJy8uXlBT9nNPpYJgTWUxlG9DPUk0HqpGYkdLPl7ZxHMdJMm4sOo7jOJ1Ka2ur1NfXm6HHVlFRsf85xmBZWZm9VllZKeXl5VKrxiBbXV2dGok19h7GJN/R0tIiaWlpapzl2Zabm2uPBQUFMnz4cDUsiPr1kwEDBtj+K1askF27dklxcbEdx+jRo+XUU0+179uyZYusXbtWtm/fLo2NajzG4bOpqalqS+6zz6Snp0tmZqZtWVlZ7T4OGjRIcnJyDjqucGzhkY19+H7HcRzH6Y24seg4juMcFgwXGFkYdhh1YeNvDEGMPQw2XiNqiDGI4Rc2/saIw0BkPwwzNgywgQMH2haeY2hhkLFlZ2fvN8bYiOg1NDRIc3PzfgNw/fr1sm3bNvvuKVOmyKRJk+xxxowZMn/+fNt369atsmHDBtm0aZNt7M9WXV1txiLHwn75+fkyZMgQM/QwDMM5Rs+X/TjWcEzsy5b4nC3x3KLPo49uVDqO4zg9DTcWHcdx+jgMAxhKRPGIwGEQRh8x7DDO2DACS0tLD9r27NljhlgwCjHY2BcDjKhg9BHjC2OPRwwpDEEMM7bBgwfvfz527FgZOnSo7UOkj+Pjt/h+DMQXXnjBtsWLF5uhyvfxeT5z6aWXWjRx+vTpMmzYsPhZHoBzWLJkiTz77LPy1FNPmaEZjh1DkN/G0Bw1apR9J4YhG+eEYcn5si/Xpb3rFX3k2nIOfE/0/MLz6OucA9cpXKv2rh+P/fv7moKO4zhOcnBj0XEcpw8Quvr2HjESMaDYiPzt3Llz/0ZK58aNG2Xz5s32HvuQ9gnRx/AcMAaJCI4YMUJGjhxpj2ykjWKEBQMJww5jsSM4NjYMNaKSt9xyizzwwAPy/PPPWyopv0kEb/z48XLOOefI1VdfLSeddJJ95+FG6YiGrlmzxr73wQcflGeeecYMQX6X78DQu+CCC+TCCy+UM844Q6ZOnWq/i6GI0cj1iV6r8Biesw9GI4RrDuF59DVSa6PXKvoYnrNhVAaDMXoP2nt0HMdxnGPBjUXHcZzjHKJlzBOMFZGJbUTpSMHkOQYgxg2PIb2TyBhbiKjxnOheRkZGrNDLkCH2GJ5jsIX5ehiCGGxEwjAcQ5SM50QJeQzzBA8VJSOCSATw8ccfl6efflp27NhhhhyGEMbT2WefbRHEE044weYm8tvBkDpcY4lzChFTooYYxRijRByJXBIxDamwRCmJVp522mn2u5wz6aPhOkWvWXjEGA3XPhqNjd4HNvZhOA7XK/G6RR+5zlzzcA+ij9HXo0al4ziO4xwNbiw6juP0UohuYeSE4jFsGFNEAMMcQh7Zh7l2/I3hGDZe55HIF48YcBhGoThLMP5IxwzGB+8TAeMRQ4nXeGQ/DMFQHAaj5kjBuCISh2G4evVqq2xKimhRUZGltmKEMv9w3rx5++ckEmnjePjdzoDrwPXDMMWY5jgomsOxYFBzjYiWck2IMjInEmN15syZFuVMjGhijHKfwjUP9yDxkftD9DTcw8TH6HN+I1z3jh7ZMHDDPQxb9L6Gv7l2icftOI7jOODGouM4Tg8EwykYGdHCKhh4GAz8zXMiUtHqojwSqeIRQ4RH0kwxCEJxlVBQBWOBx/AeBmAwIthCkRaMIww19iOyyGNnRKwYfjBUMYKIIlJ0BuPwySeflHXr1tl5cXwTJ06UadOmyYQJE+Tkk082Ay0cT1fCsVG9FYORjXTVUEiHDcOM+Y1EG9kwHseNG2dRTqKQR2KAYVQS2YwWBQrPo6+xRdtDaB/tPTK/EYMxseBOeB4e2Se0iUM9+nxJx3Gcvocbi47jOEkEowBDEEOEVMXwSCokG+/zGgo/RlRIUwwb0S3m6/EcY5LP0I2HlMVQCCUYdjzHGCD6Fk1TZAsRQ4xE/sYoSEaEiXPk/ML5UM100aJFVnAGI4z3w1y9WbNmWbrpWWedZcd6NBHLzoK00RdffNHmNbKRshqK3WDQzZkzRxYsWCCnnHKKzJ071443GGOdZdhyr/k9rh2/HX0MG3+zcY1pS6GNRdtb9DnthAhjtG1gjLNFX+M8MJCj7Sw8Rp+TLutzJh3HcY4P3Fh0HMfpRKJdangefSTahyIfiqKEjcgaxgfRJf7GUMQQTCxYElXCUd7ZUOqjRVB4ZD4dqZo8Z5+ekGbI+bNhxDAv8J///KcZiRiLXBeMFs6FKOJHPvIRM7j4GyO2J8I5hMI4Dz300P7z4x5x3BTFYaP4DhFHXg9bMiB1NxTaYQvPE1/jfhB9DnAO0UfgOU6F0NZCOwsbf4fXQmprOM/o+bb3muM4jtNzcWPRcRynEwgpo6SGRouX8EgqI8Ygz4kKYRiF6E4ohhIiQETV+JsIDWmCRKfCRnSHlFA20gJJdSRyFaI9IaoTojwh0nMkBV+6As4npJree++98txzz1k6J9cF45jzPPHEE82oOv300y2Nk/PlvDBye2rqI/cxpAVjdFEUJxTGIUKKkY6BxX0idZbCOGykrCbDAA5LoYQ2Fn0efQyRyrBF2270ddo4HKr4Do+0TdaqDG02bOHv8IhRSRt1HMdxei5uLDqO47wMKNQYBhg8bCjXPBK5weDhPZ6zYfxgNIaN94jc8Bn+psvFCEJRZsPY45GIDQo00bUwnyzMF4tuGB9sKOb8jbLdU4uTcM7M8yMCR/omxiLLcDAXkWMm2nbxxReb8UsUlDmJRKY4r94E9xRDn0I8FMYhQsx5UxSHjTbC/SLtl3MlchoK41CkB+OpO+HYaaehvUbbbvR5MIyDDIQtKhM8YoRiNHIfaeuh7SY+px2HeZNBDqKPYcPodqPScRyne3Bj0XGcPgtRPFI9MeZQiMMjBh+KL0ovSjJ/B0U5urEPhgARF95nXyJ4GHkouSjDGH8oxhgEKLy8F4zDqELMnEJeY38UaBRqFO7eBEY1148o27Jly8x4eumll+w5xWs4f4ylyZMnW9EaDCXWMOR6cI162/l2BO2BdhEK47BhJBNtZL4p9xjDOBTGYSOaOmbMGGsHPdUwCu2ctt9e8Z3wnH2iTpMgV9HnPCJ/ob0HOQjPEx+RpajzJDxPdKgcL23IcRynp+DGouM4xyV0bSijRE0w+qKPKL0hmoJyi2IfTbdj6QTW10OpJQqGEowhRHQIRT48sqHsEiVDScXgQbElQkiVTBad529S8jAIoootnzse4DpzLTGcMRKJHi5dulRuvvlmu3a8x3lzPUg1ZWNxeyKJx8s1eDmYG8jyG9HCOKGCLW0Uw5miOGxEHImu0m7YaFe9bX4fckV7QJaixXfC8/DI+bNvkM1EOQ2vAW0IuQoR+PCY+Br7Jcopj4mveVVXx3Gcw8ONRcdxei3R7ivxOUo4EY5QyIMoVyjoQUEVojworSEKEiWqnPMc5RLFnahYKOjBRjSI6BjRDwxDlFUMwb5AuN5EZrm2d999t9x1112Wcsp15bqRPkihncsvv1yuu+46W1rCEZvX+PDDD1txHCrAEmXjerJhVFP9NRTHoV2FaFlvMxpfDpwxtBVkMshmkNfo3xjWOHiiMg7Rv3mO0wY5TSy+w5b4GkYlJMp64Hi71o7jOEeLG4uO4/QqMAJRrolMUIAjWoSDqBbzBolYBEMwRCdCQQ82DBweUQiJbrERmUgswMGcOpRMIoMhYhE2FHiMyLDmIH+z9YWIBUo+BuJjjz0md955535DnGvPdSKKyuL55513nhk/XMvOXDi/t0P7DanNXLvFixebAUnUkXbMNcTQ5ppREZaiP0QdicoeTyDLoQhPooxG/w6Ryo4K74TXQup4opwmPkduub5RWU98HjbuhUchHcfpy7ix6DhOjwDFESOOCEKY+4Tyx8bzoCDyHCUbgyW6hZRSPs/3sIVIQ2LhDKpToiwSESQayHyokB4a3Xif9zByUDT7Mhg4RGOZf7hkyRJL02UeHhv3jqgrC+azUcAFIxtDEWW7pxbg6QnQXjEY2ZjXGS2MEyqqcm2Zz8hcz5kzZ1qqKs8xbvoCGJTIMzIelffE59G5xaHfSHzORv+C6hMtuMMW/o6+jvzTb4S+I7pF+xU+40al4zjHI24sOo6TFEKEAKUOwyM8orjxiNJMuhnKXjAWo1uIIKAQYpwEZQ5jJDxHaQuRwhCdSVTwMAAxFkPBDPbpK3PnjgSGBu4ZinYwYCjUsnz5cqtsyj0g8oIRg3HIHESiX1T4JBLrHDkYRERouc6rVq2ya878RoriMI8WowlDkcgtj6T1kgrNxjXv65Fbrg/9SjAIo/1H9G+eB4dT6IuiW/Q1DMBQZCfahyT+zRbti6LPo38T1XTnieM4vQk3Fh3HOSZQ0EgXSyxSgeLLY0gzQznDGAxGX9hYaoCoIUYgf2M0okyh+IaUMR75OzzHKExMF2MjksV7KG6k8KGgOUcG1597gZEYloFgPuLjjz9u9w/jmmgsBgtpkUQSzzrrLLvWrgR3LszZI5JLeipzQZlrG4weIt2091NPPdWMdO4Hc/GC8cL7Pu+ufXB00B/R34RiO4nPw9/IQmLfFp5H/6ZvwmCnHwoFd8IW/Zt7g9FIPxa20K9F/0aWPFLpOE5PwI1Fx3Felmg3kfgcZQpjL8xbCxvpibyG0cHfKGcYlYcqIsHfRPyYK0hhlFCYgjQ8KkbyiDHI664Idy7cSzYMEubPPfTQQ1awhvuGwY+xznU/99xz5XWve53NSUT5dZID0S4qqlIUh40IL8Z7uG+kpVJlNhTGiS7B4bJydHBdMc7pvzoqvhMekROMUD4TiD4H/sZQRG5CwZ3EwjvhNQxPMh7a6yMDfl8dx0kGbiw6jtMhRAeJYmAMhjmDPDKXasOGDabAoiShUPG8vSIyIbKI4oOhxzyrsKE0sVA5FR+JWPE3j8z/CZHFUJQi/I3HnUenc+DeUJiGlMfbb7/d5iMSUeS+cu9JdZw3b54ZIkSxuFcosii9fX0eZzLBECHqG1IoifhyrzDs2XDaIGPID9ErIr7cr9NOO81Sg/1eHR2h/0osvpP4GvckGpUMW+g3w0ZqK0ZetG8LW/Rv5CtkULRXeCe8TiTZ0+gdx+lK3Fh0nD4Iyg3KJcYAkb9QACKkYaGU8ncwAsP8nbDxOgYG30OqFAYcEcGwriDKS0i94m+UHpTYxHk8GIUhLYsoSHh0j3nXQ+XY9evX75+DyLw45srRFrhfGBikN86aNcvmJTIvjogHiqzT/SC/rA+KYY/zhjmOL730kj0SHeZe4Yjh3hGV5z5yTzH+fU5p50NfSN/YURGe8HdwwoS+NzxPfI30VvrVxP4y8Tn9Ln0o/WzYkN/o32z0vzjcHMdxjhQ3Fh3nOCJ4ulE0gpEXfUQR4RFFE4MvUVEJcwpDRDEoLMHIC48oHmwYdiieKCukvQVPN0oJfwdjkQ0D0ufgdB+0De4phgVGIgZiKKSC4cj9JM2XQjUUT5k9e7ZFpygGFNIZnZ4JUUfuIfcyGI3c51AYh2E+FMahmiopq6EwDk4djzomD/rWkI0R7XsT/2Yf+umosZnouOPvkCKeaDCyhdfCYyjqFQzOaJ8entNPu7w7jhPFjUXH6SUgqigG7RVYIBLIe8EIZMPoi6ZAoTSy3AGvsR/fF1KeUA7CI4pHSPVEgQhFG6Ib82owLHifv4Ph6PQsMCJQKFE8iUJR1ZQ1/e677z4zJDASiEJgEFKs5qSTTrJ0U4wJT23rneAU4H4//fTTVhjnueeek3Xr1u03RpDrCRMmWNSYDeMRJw8Ghctxz4E+HecefXdi4Z3oa+yD4RgdE8K4EP0biEYm9uVsOAxoAzzi5KMthLEhuoVxImyME47jHP+4seg4PYj2xDG8xoCPYkAxhWgxGdLQQpVEDEIUCJQH6Kg4AkoDSgEGX7SwAoVlUCRRGkORBU9d6p3QbnAKYCw8+uijcu+991o0kVQ44B5TxfTKK6+U888/39ITPf33+IM+g2hjKIxDkRwyCAJEkmkHFMU588wzrcIq7cDbQu8Ao5L7yZgQLcKTWJAH4zLI/qHGGRxIRCDDmNBRAR42opWB9tqLtyHHOT5wY9FxuhmMQJT6aAEZHokEkVpGZAhlgDmCPI96i8NGahORRaIKeHyDpxivcXgkeoQxSKoR7zPQR73EbHiO2fAYozSw+YDfu8BRgHHw2GOPycMPP2zPQ8SZe0864oIFC6yqKRFE2gbKoUcSj0/oFzAoiDaykYLMMhxEHtnoU7j3OA+Yczxnzpz9hXGY60j2gNNzIXuAexzGgmjRnehr9AtElhlbwpZYfIcN5wLjSRgTGAOiY0R4DSci/UkYXxK36OtEKh3H6b24seg4XQjiFYrFMAiH1CEGY1JCeZ1BHCUOgxDFjb/ZeM77DPZ4hHnEkMPIixYwIPqHIUi0EIWPv3k/zEMJGwM7+/MdPi/l+IJ2smnTJoswE0nkkaVLmLdGW8M4JN0QQ5EF9IkOBMeBp5L1LUIGAhkJpCIzv5FKuGxEp+gjKIzDNmXKlP2FcdjoR9x51DsJUxjC+BLdooV4GHdCynLixhgWnuOAwghNHGeiRXjChjMqjFftFd8Jr9MfeftynJ6HG4uOcxQgNhhvYXANAy0bAyoGIht/YwiGATZqLKLI8xrfw/dR/IWBlQEzPDKA4sllAGbAxRiMDq7hEWMxRAfYzw3B45vQ/og8s2EcovTzSHVTFEOix1TCxDg8/fTTTeknxZD0Y8cB2hFL4DCXFWORgkcsyYERieFIn0J6cih4RIo6RXEwJOlznOMP+g6MxzBmdWQ48jpjWxj3EsfBsPFdtLNQZCdxi77O8zD+hS3xbzbapRdLc5zk4cai4yTAYMmG1zSxSABGXtgYKDH+ElN5iBgGDyzRQvYlbQcDLprKE6J7GIIMkihf0dQdUsJQ7jH+QrSQzzh9FxwQKGW0LyJDixYtsnRTDETSlDEEcRhgIFKshojiwoULTeHyCKJzKEhnpA2FwjhsGI7BMKBPDMupsFEQKarg+9zmvgXtAWOQMS86DgaHaNj4m/YTHUcTx9WwYQDi5GL8C4/RLbzGWMi4GcbSxLE1bIy7blQ6zrHjxqLT54mKAAoTRiAbg18oJBMeSe1DgWI+Ifskik9iCg0DFcp7KAwQLShDihfGIINfiBp6Co7TEbQ1qlpSzfShhx6Su+66y5QwnBoo6rSl17/+9XLeeedZFIh25ThHC22LaPX9999vhXGWLl1qjgraIco5BU4uuugiK4zD/EbSmr3/chLBqKTdtFd8J4yr4W+cFYm0p6LS/nCmhrGUrb1CPPSB7TkxvJ06zpHhxqLTp8ATyoCExxNjMHhDidIwjwcDMBiCRHESPZ94RIkUYlQyCDEYJRaSQWknEojXHU8oRmB7Xk8+z6AXvJ8e+XESoc2hSFHJdMmSJVbJkhRB5gsRXSRFcO7cubbcBQo76YG0N7zutCvHOVpQ8ukDMRrZQmGcp556yqrq0l/S1ogs4gRjTmwojBPmwzoOKibjJX1Ze8V3ohtGZTQqmbiF8ZrxmTGzvXE1upGJw1jMuHyojX08a8dxOsaNRee4IKSFhvTP8MjAgnHIvAleY5/25laE1xnMUJIw3FB2MPjC3EA2DEIGFuZNMMi0N5+C/Rh4MAb5m0HLcY4EUplRzkkvRUHHSMTzjpFIG5w/f74ZimEeIkYinnRva05XQR+J4wLHGvNkQ1Ecqu2Sbo+DIhTGIQ06FMVhviPveTqg83Iw/kbH5WjhnejGWB3G+ENt7EufSPtLHKfDxnuM4zg9omN94sZ7ZAC5Uen0RdxYdHo8wcPNAMEAwCPGIY8oKXgkMQgZPBIHEDyRvBYm7DMYEc1LNPLCvJvgieRvtjBIhIEkeCEZYHjP01mczoB2iWODiqakZmEkUmyEwiMYjrRLjMLx48dbiilzxjAWSb2iPTtOMqE/JsKNocgWqu/yGht9Jes3Yiiy8Tw4NXB2OM6xQH8ZHeejG1Hw6HMcbInGZuKGDkE/GozFMPa3txFNjxqZUT0ibLzn/bJzPOHGotNthPWhEie7h1RP3uNvOnMMPtKe8GhjAOLlJl2U6AvvYyzyfSH9JGwh1ZNIIX/jGaQISDR1lFRRKvzxnPd5ZBDwtFCnK6G94vBAmaEtMyeM+WEsfYHCTXulLZLiR6SGhdMxEknxc++205Mg6h3WbiRNlegjfXLI2KDQUkhRxdmB0o1CTn9LO3ecrgD9AYcyjrjEwjvRDf0iGI1BBwnPoxvqMoZgSF9N3EIBHnQM2njQQ6Ib+kh4zlQBny7g9AbcWHSSRmJTI9pHRx6d6M5GJAUvNYoGf6N0YDxGSYzoBSMvFJDhkagLi0pjCNJx8zoKinfOTneDLKBEYxg++uijcs8999g8MBQW2jbtmcIhV111lc1HJIroUWynN0BfTSVViuI8+OCD8uSTT9rrtHkUZfpm2jbFcc4880z729u2051gHGJMkmYd1UVC4Z2wYVjSRyeSqNvgaKYPTyy6w99BR+E5FdDbW8rI5cHpabix6HQZKA2kKBERDBPTeY4xyCNpIlFvXtiiRWT4m46X1A+MQbx24ZEOl3XkeM77GIREBKNePCKLPIbIIo/eETvdBe2d6AvLXbzwwgsmHzhFaO+kL2EUXnLJJTYXkTXtQpTbI4lObyE4QkgBxBlIlJw2z0Z6NX0/Tjs20lIpjEMF39e+9rXxb3Cc5EKbJbU1URdJ3Ei/Rm+JRiXb24hUsm9UF2lvw3lC/05EMug10Y256RiVjtPduLHodAmkkLLo/Gc+8xnrPFGSUSB4DCkfGIMowWGeQNhCKgfv8TeGIF66MB8gzBPgNRQOnofUDk8ddXoypJz+6Ec/kr/85S8mF0BaHsYhc7swFpnfheJAupO3Z6c3Qx9PBglpqWxkjDAXFycJxXEYC+i/Sa/+whe+sL8YjuP0RGjPOPbQXzoqvsOGroMTEMOSNh7mULLhRAmvMR7gHA+6TeJ25ZVXymWXXWaFohynO3Fj8TBp0cu0t6FZtlbXy86aRmlo0UFQt/rmVmlq9UuYCMbizq2b5Qcfea/UVVeJDEgVSUsXSc8QyRykzzMkNVMNP4sI5qnBeMBYLCiIGosxYzAl5fic15I+oL9kpw6Q3PQUyUsbIBNzMu35gF4Q/aTVN6kMbK1ukB01DSYfjczDa0ImWsXFoi0NDfVy629+KY8/eL8MGJQreSNGyay5J5qSPG78OMnPb5uS1NcYoE0/S2UiLy1FcnUbMTBNxgzS/qJ/78gIaNaGX9bQJFurGmRnbYOND3U6TtSrrPBeX6a2tkaNxqL9hXF2bVovdSW7ZOjgArnmfR+R6dOnu7HYAZkp8bFCZSJPx4jJuZkySP/u3wvGCpp9o7b/Lao/MVZUNrbY37XN+qhvHm9aaGtrrChf1Eg8sFGAJ2YsVqnhWKcGozTW62DaoI9s+ty2Bjnpostl4ZWvlBlz5sW/2UmE5j8wZYDkIBeqQw3NSpMJ2ZmSqgNJ7xgxegduLL4MGIll9c2yq7ZRVpXXyKKivfJcaZV1duWqEKAgowg47VCrRuLjd6v21KhWkSoAqhxLdp5IwbDYY8ZA1Qz79vxBOrgR2rmNy86QsYPS5aIxBXJCwUAZqa8VZPRMA5mBn0G+uK5JdurAj0w8XVwpm6vqpUrlorS+0QxGZMdph61rRSrLYnLAlpYh4ssK7AejcIi2/XE64CMT8wuzZeHIPBmtz4epXKT1QKORlt6q7b1EZYKxYvmeanl8Z4WNFbVNLWo8NkuFbhiMToSyYpFdW2LK8byF8Red9sBAHJmVbmPF+Ox0uXTsYJmSl2XjB+/1RJCJam3/jBXbqxvkkR3l8kxJpezQ5+hQpfVNNpb0WR9Kq/YHGInVFSI1ulVXxp5X79W/9fmUuSJjpqg1lBP/gJMIw0FhBg7FdBmrsjFb9acLRhfYeDFcZQMni3PsuLF4COjAqpqa5c9rd8vf1xfLcyVVqgS3xN91nK7hbFWM33nCSHnLVDUkeiA4R55V4/CmNbvkzs2l5jDp6xETp2sZmpkmb1B5+NjcMWZA9jRwjOAouXFlkfxjY4m8pMZirTsRnS7mkrEF8t6Zo+SVE4bEX+lZ4Ch5dGeF/FHHinu27jEZwYB0nK4EA/ENU4bJh+eMljmDB8VfdY4FNxY7AG/YirIa+eHSbfKCDvy7ahqkRgd/7+icroZUI9LuTh+eo53dGJmUmylZPcQ7tq26Qf66brfcrgrx+oo62dvYLC2kEcXfd5yuIKV/PxmcniozCwbKW6cNl0tVSSbK2BOoUBlYUlxlY8WqvTUWRanry9ESJ2mQmTIhO0POHZUnH5g9WsYNSpe0Ad0/VtD2mbLzu1U75T9b9simqnqLrHu2iZMMiDYW6HgxPT9LXj1pqG6FMmpgz3My9iYGfEWJP3fiMPgv3l0pP12+XR7esVcNxUZLH/JuzkkGzOHACGNux3bdRg6MpaQyv7G7YN4V6XX7IydlNVLZ5AqxkxxoZzVqgO0m9bm2QfapMjBelWTmbHUn5aoAP7SjXH61osjSsUvrm20+u4uFkwxoa0yHKVIdhfGCFFVSUlO7Ma2dY9qqxuENy3fIHZv3yJq9teZ8d5lwkgVtjcyOknqmyjRaNtQUnO69ZI5vT8SNxQRQip8rqZZb1hXLbaoUV5M2EX/PcZJFUI6J3on0s3kpIwemd0tHRzR9j3a6/9xUaulEq8przaB1nGQTnBbMASTaPqtgoBWD6o7xn2N5YleF/FXHiru37DHlxKXCSTYt2uhw3DFWEFEZPShDCjNSu2WsIHK4o7rRdCfGik2V9SYnjtMdMD2GebG7dcxIHdBfpuVlSYY++vJpR47P/EygWBvV/dvKbPCn0ql3c053QcYOHrG/rN0l/9m6x5Tk7qC6qdVSsm9cUaSDf53PT3S6FRwVS4orLcq9dm9dtxWNobjT3Zv3yIM7yr1wjdOt4NCramyW36zaKQ9uL7eISndA0ZrnS6vkNyqbO3W88rRTp7vBWbF6b638eNk2WVZabfqMc+S4sZjAI0Xl8uiOvVKkioDj9ASIWNy/rdwcGN3B5qo6+efGElldXuOVf50eAQoAUYsfqgKwu657nCj3bi2zyCJea8fpbjDLKMBH0bFHVIfpDhgj7txUKusq6mxpDMfpCeDgplL1r1YWyfqK2virzpHgaahxcICR9vfzl4rkcVUAukMpZt2kt00fIT9dOFXefcJIMxKIdHJcztEzb8gg+dLJ4+WLC8ZLfkaqrYFGasLRcNX4IfKFBePk/bNG2xo+eE+TcX8oDjAobYCcOSLP8u6TlUTB/JMndlbIz1fssPlZ3eUnvuHsqfIlvX+Uix+o58+cSQ9wHhss0/LZ+ePk3TNH2rzYp3dXHdX9ZY7UKyYMkW+fNkmuHj9YBvTvZ5HorgaDcUtVvZw/Ot9KpCdrnhbREpaG+e7SrfJ8SbXJSHdwoZ7398+YLJ+YN9auAcoQ18M5NqieeOcVc+RqbdP07ywJdLS8fspQ+eW50+RilbWnd1da5K2rYcoA1YNPHpotWSnJm9NLJtZ/t5XLH9bsSsp5tgfz+j84e5R8Xvs1Cv6ckD9Qj6ks/q5zLHzyxLHytVMmyCnDcqxo0dFCAb/rVcf91EljJVPb57PFVfF3uhbyBAkCLRiaIxNyMru1BkRvxK9WnIbWVuvM8YyxFEB3MKtgkA0qJxRkyXzt6C8fWyDT8rPi7zpHC0UwpuZmyUmF2TJ2UMYxDaCs/8YANFcVChTUNFYQTwIYpCiCrN9G9dFksbGyThXiKlsXqzuMM6r9Ucns3JF5MleN/gUqF2eNzLV5as6xka9G3vS8LFWOB5pcHO0VZV1ElNPZ+j2sETo0Mznrg+ItRjFeVlplFUiTBcU6Ht+5V9bGC3d0B1xv+rNzR+XLDB0jzlGZmJqXaY4U59hgvKCfma19PDJyLAyL3yeq+DJXKhnQJjdW1spK1WWSCYVslqosdtd0CWBcOE9l4mQ1CE7U8WKhygVLUfWUauK9GSq0MwZPzc2Mv3J04ExErz1xSHZSK5Siv+DwXlFWLdur3al2pLgExWHOyZOkFKnS0V159hNzMiy6iIeYDh/BnKCvOQ6Uart4QQ23ZC7fsqGizqJE3VWkICdtgFwzsdCUthDZnJE3MGaoU83B6fO8uKcmqQoqa8U9UrTX2mN3jRWUhCcClj6gnxU2GZgywJyNjCGOQwXIFSoXyWSVGqfMIe6usYK19SblxnQo0nFx+o/MSpcrxg3u9qrJTs9hTXmdbK3yaWZHihuLcZpa9sWWA2jsnqhiYWaqjBmULhna4W2opDHX22usoXSs3k3n+ID02RXltaqgxl9IAjtU6dhS3T0dK2vrsVYSSjEKyPq9tTY/DDlZOCLPoo6Os1plAkdKsmCKAoUSunPRfWSCaBXp9Hdt2SOsAXzK0GyZX5gd38PpyxBpX2OVtJPHFlXAWb6ju2CsuGTMYNObVmqfwJaV2t8ii4wVvmSCA+jXXpPkyHFjMQ4pTRu1c+2u+YEsHHqWKsAs1fHM7kp5rqRKmAqzUDs6UlMdx4zFsuqkRhZL6hplVzd1rMFQZH0kqvs9WrTX5ALnydtnDLfoouOwEH5JffIii/UtLZZy150VUImgsJzO8j018uVnNlkRKiIqpNs7DkWfaKPJZFdtg40X3cUwlYfXTC40w/CB7bGK9kQXcaqMHpQumT5HzVEwFndoW3WOjH77lPjzPg0esQvvWGoGY3esIffDMyfLNROHyLLSGvnw4+tMEfjxWVNsbsqf1+6SL6lC0BFvnTZczh+VL+NzMmxeBCmspA7i+aZwRa52nm97cJVUxKOmefo3Ka5vnT5cJuVk2hy+6qZmG1z+uHq3vHfWSItwUhaetZIOBSker5081BT7jz6xTt48dbicPjxHFfpUaWptlW3VDfLzl3ZYxPa0YTlyme7PPI5mbXZ4dzAA/r6+2OYehcvOOUxSxef9s0aZAkQqYj/9hyHP9/16ZZEsLa22VJMo4ftPGZqjx5MiTfHfYOI9E5p5/5criuT3q3faPLzAVeMHy4WjC2xQCdGqvWqYPaVG+z1by2Txrgp7Dd4+fYR8YPYom+P1/aVb5eZ1u2V7kiJvOEZJwXz61fOTllbzhac3yk9e3N4tc7MWFGbLB+eMljdNGSY/Xb5dbt1QLNP1/L99+kR7/+r/LLd5xu3Bgu2XjC2Qy8cOliGZqTYfD4OTlL31e+vkcr3nK1VGqPIa/Y7Ltf1QxGhaXqZdY9oYlf34LG3qpCHZ8q5H1siLe6rjn2gL8nax/vab9bhXqUzRBpnjSltmUj3t+LniKqtauKGy3opZnaptc3BGqlUQ3FpdL39Zu9uOK7EQ06smFsqVeuwHFqTvZwraEm3P96hy9HRxpRXrClBMgO/mnJhPyLGF3yDdHa97QUaK/GtjqXzsifX70yqZS3LWiFz73NjsdJXJAdovtloEj3NnEfoQRcCT//rJw+Tj88ZYH4N8/mz5DnsvGTBP6efnxIqCJQP6HsYKzjXZy8jQjrmfX1gw3lJRuW+fWrxebjp/hvVh3P9vPb/FHI6J8NmpeVmW1r1Q722h9sPU0KRP/femUuvDr5ow2PrGHy3bvt/7TrvEYYlsjFOlm/XK6M+RC6L9FIyo1Tb9d5XPB7aV22c64jWThsp7Z46UF/Qa7lEDf1r+QMueSdPvZOmHxXrct+p4kKuy9hYd107Qc0zvr++pHBLNvWfrHjUEDv4NjATkgmyD0TreZeh4xtizp75ZHtPx5Q6Vs0TDCTmgKNBrJhWaIYFckl7MveW+fvWUCSojDfJRHYtv1z4CuH6D4239vFF59rukwje27DN5opDKnTpmMpYFPqz917dOn2Tp/K+6Z7lds2TAWHG2Xo9HXnli/JWu572PrlFdZXebcTlZnKlt+l+Xzba2+XHty9B/aGu0W4ruoIfQhhJhnu9s7Ruv07GdPpq/KVrFWPHIjnK5UNs+/SgyEtWH6IORi8vGFWh/GUv/xmB+aMde2VXTKB+eO9pkKegrHcHvfVfbCM5P9qNfvVbbc4HKHY5hUuzRk9BdeP3iMfmmH6XoTa7QNkt/zHElFhXD0UoRLAIOYW4gcsr3L9Lvow9PBF3wI3rcjHNDs1IP+g3knPntTIW5+M5l8U/EoF9BX8OxS8Ezio3xW7R7jo3rvlfvC/D+7XqfGN+vV93i86pjJBMizJ+bP06+rjLuHD5eDTUO1bt+sWKHDRTJHP5JtRuXnWkG34isdHlGB/u/6WDJfBgUuSmqtNLpPauvM2BGjw3D5jo1+FCmmdDNYI8RlqePFDuYUZBlxQ8GZ6TJb1fttPQpBv5zRuZbxbBz9ZHvYGBB8aSjRNiZII6BRmdJafhDcfrwXFVeh8hJhbFiAKepoThElRDmD2AU0onQ6WGose9wPUeKwozQzmuCnjfGMArXcu3oUFQ5Poo2fHD2aOuIGdSZEJ2mnQ9K/wxVLjCKg+EYDHsM1DdOHWZVGSerYcirfIaF7OlYOZZBqSlWeYsOmSp3nDPH/ja99lyTEapo8HVcDyZzY0hzLgx+27SD5T0mZVMNjONijivHnczKb1xLqleiXCUDjGWqAydZJzZO1MGEe8Ng+jM1FqnWma3XfZ7eAxTMIh2QuY9R5QxYqP1aVUrfNHW4zBkyyGSMQW+IygEKNrKBvCBjpJ5TxAdwerzZ2kKetVMUBn4PRwPfiaGKwYVCRNvrCNoVCj1KOceJUoFSrIdhc8vG6d/MRaYdMyifoUoOhZM4TxRQFg6mai9pXdFKjO/Twfgt2sYZtHNVxpEXqsmhCHBOtFkUpVJVkjl2ZJviGh+bO8b6Er6bY6DtIOtj1Ajks8gQqZz3qbLLbR6rx4eRTfT2DJVZjgtnDVV4SQGeodeAc8TYJILB+8yXY19+FwXpmSRVuAOO+YpxQ6woSTLAiPq5Klr0tclGb59cN2OEnKPGCvPSbt3Akja1dm/pa3kkO+XxnQf32zjgJmo7xtFFHzlR7z9GNmdg7VH/Jo2V/rWqqdnW6iMqQ9ulKui71BCnj6cPpaYXbYuxhfGCNsYi10Q5X64KLkYWx0//yW/hFMURka3jDbLJ79Emkc9ThmXbMdLukBeOmX2fVIMSA41jx+mJA+91KrsUHsPJyeu0Sdo4fT99JuMFxl8Ao/V/9HOMSRim9G+MX8g658V1REm+T/s/ri99CP3He2eOMgNzliri/AZjF9eCz4UKizgZA4x7GBv0NYzrZfqYLMaqrL5dr3WyYCF+nADdMVZQqIt2jQOACsX3al+2Kn7fqKCNscd4jU6TCH0jlVMvVCMMxxfGBP00DmGKdtG+aQ98H/oQ79NH4syg3WGU8TuADofeME9lhTEA3YHfjba9RNCV+H0KKtH+kSn0IwqH8bvjVU8ap8eCLsM5Iq/0+znaXpEZZGOvtlX6YtoZn0OHQ6e8QvUbshDovzlEnB38xqScLNO9kNeQIcH5vuOEEfIG1SeRJeQOZQi9kr9xGjIeMkXqTzoGBmjj79T+gWPjWFFNuEYEDBiX6Hc4Lhyu6KDI6Ov1NzgfHKIPqUGeTGieOMtoK87h43H5ODQgBqBkd3QM4njw6QDwIJFnzzEQycEbiiFC0YLTdFALHRLEBv8MGygZuNZV1Jr3DKMQTw6LkLJPoQ5wURB6hPoc7Uzwht62odg8X3zm0aIKM4RQVI8UOq8LRhdYh4oyzXcSMYSLVCjpkBlYb9HXfrNyp9yyrtg8XAzM5+n7dHCAMo737Mpxg83Te9fmPXZOLDbM93GN+K5X6z5R5RDDEi8a3/OvTSW2PxFEPL0o3Biakctn1xJP3lumDTMllzS2W9fr5/S4f63HR8QJxZfvpBpnXy2mgkGS7OgJ4DRA0SMyvkmNOTyUOHKI4uJlRTlF8UQhTARF85XaxhlEWQvvD6t3ye+0LfxD7ymDIxHkgTpgBWgLOBO4z6cPyzXj808qD7RTPovxg13AsRxpK8C4wluL8n0T36ntklRaHDoXaZslSsgxEUlEZm7fWGoDLW08FLfi8wzIb1HjFwMaQxV5pZ3ymftVMWJOJzKAAlMYl1+URSKDKKtcO9Ky+AzXgvaNM2WQDv6JMO+NPoJoKNV3/6qyyu/8Xo8dJYwIK8o2RjXKVV8DcSA62w22ojmyuD85qsit21snS9Qo5zAeU+MQRRinA8ptInjzMfq4rxTFIWLCPaU9YHAyVmBsJoLj4rV6r1mGgd+indIf03+jBI/SNoYyeSQQKaRtMsY9tH2vykXs+xiziPJxnIxtD+t79OHIIQYARhkKONFGFE4MTuT1DVOGqoKdZUban9fE5Ai5/c/WPXY9OGdkDaORbhwD8+oJg21Jpe06Bt2k+yKXKMCMuSjRiZCNc/HYfDMOUOQxin6/KiaDHCPjHkoxcoHsYmz0NRgnumOsANou/RG/T4SXPhJdgYgY89xppxhxjCtRMGYuHVsgF6g+RAYSYwRtgTaJYThaDahYpOxAz89z9IJXaN8alhwKffHf1u+2vpa2daRQsdUqSetPkbGEnN2scoH+QpslMIBhyTrg9MV/0/dwdmKgnqU6TEhBxxlIpJ2sL4xk+oYg68gFn2FsxcA7V8dQrgmODwxj9EnOGVm6Sdv1b1fFzmmb6oo4bugnojC+kcGDzPK7RPF/p8fG76GvUfCI+4LjFIer03txY7GbQfhR8hBYOrZoyiPRxA1qBOJdukA7p6jBwsBHuiXGVXlDkxmKpAwS1v/281vkx8u2WRSNtWWinFg4yISXlJ8btfP47gtb5YdLt+lntsqX9PMMgkeTRkInvb2m3lI9vqO/z3fGUplikQcKpeBZ5T1SCkmVsoWD9ZRi3rzY96AI4TVHkbhFOykW3ea7OB+OlQ6MFA9SK0gvQbFmIOdaoPCjwHz+6U2WIvoD/dzXlmyyz1jEOHIpUDxmaedI6hIRAjq2Lz+7yX6PKNbXl2y2whFcc4wSvN3JiuY5MacGaSrcs0VFFXr/Ym2SuVms8UT7wJPPfYmul8TgSKoMEXkcKMgEbY628H/PbdZ2sc3S06JRIYwxlNDZBRTSabUB71OLN1g7pc3xWdYPOxo9CGMbOaYd/kxl80favlBii+sazVijEAXtLfzO9foc5QBlM5R7H5yRYpEdDGOWT0Fh/8oz+n26L99Hm8WpgiGBAkMEhn6F/a9QY5RvoX0jE2zI00efWC+LVIkg3ToKn8MwJXpJ1PZ7elzfem6L/Q7H+F3tJ+ijUMJJ+yKi6CQH+qLRAzP2G1pr9taYEgc4HFjKA6cBEQCMrqiCy/1687RhZtg9qEYYbYb7SRv6tLZ1Ui0TK8oiFyjhOChwoAQZ4pEx5rNPbTRlkPTkI4E+O13bGe2Y4/ixjhM46XDs4CDEmGUc+pr2waQz0/Zu31QimyrrTXFHcUZpJQKDwxCn31qVdb7re9q+w2e+/MxmGwOJcGDsEgHit3E0kiGCDP5VlXL6CMaXb+jvIfOMIYkwBhOJJKL0QmmV3KD7fVXHC67fd1QmfqNKNUo4MkdWDOOzkxxoCxiD43IyrJ3+W/vvsN4oznacc9x3+jQicVGYvoM+xNJpfI428H2VC+4p4wZTBYi84ZwMIFevnFBomSE4MX78Ykwe0L2+oLrH79UgO5r1TjkPsklwtofv4xhwNDJW4ByhH2daCBu6FrpNZVOznT9po/QRROtJQUbvWrRzr/xI5RXZoG2j19yossYUDFK2mTaEYyZE8zGo6VPY96vPbpafqGx+S68Fv4cjJUxTCBDEQIfC+YrcIEvokuzPb/5Cj5FoOpFOxthowMPpXbj2283QQZD+QwfEYBPNbWfOIOlcpITOVyMvKmgIN+kVdA7Mf2IgjYIhxnwNUpWiYJSSbkOn9I8NsUhdoFIVEDxWR7MGDfnp/9lSdtAEdxKFUCRQshfp8aCEB8q0UydtIjGFk4Efzy5ecpSIaKdLWhQd6DI1qnNSUyxtAoUaY45BnE5pjSovKAGhT+P7Ub7XlB/c0ZGKcY12+BgXeIVJNeL7Aww6RJS4JxzPO2aM1GvuBVWSBeuMkiJGm7xjU+n+uXuksRCRXq/KIQow3kqi8gEGPKIh9S37dECslSJt3yFVmc/SnpCNioYD7Q75eqUqkMgG0Q3S6aKgbNy/vcw+mzBWvizbVf4YZEnZDB9FoWcj3e+BbWUHtTsiVhjERLUDFlnUa0GqHvNmyAaIQvsl7ZD51jhB8JQTaeH5SJUnDFJkk0hUlKd3V5gHPso0/RyfVbEyA+R+VVRwtASQre++sM0MSSKPpCY6yWG49vmfPHGMpRsTEUu8d1QtRjZIWfvEvDFm4AQYZzBkqrWftkJROmZEQSaYxxRlZkGWGWaknbI8CVH9UAGW9svYgoJIWzgSGBMoXY8iy/ECzhvkgD56e3WjtdVoejnZBYkK+Okjcs0ILKtvlrt0bCGdHBkHZJ50YQqdkEFDn4BcoPCQRo0D5nkda29WxTzKC6r8J74GKN4Ym4zAzKlMXJT8nxtK9s9tZH0/nC5OcqD/J1tkcHqqULyEyvYB+l0caThXcG7hJIiCw3G0yhVpohg4pEsGmF/6uac22hrDUd0Bw3NcTrrqNfss2v6w9pFRWGuSMetoYP3Wm1QHC3A8OPaRGY6D/p/2DOhu/9xQaucWQOavnVSofUWaOeoX76q0+exRiEwStUSfxBHENCWcQlwf5Bq9MaoXMiYhXy9pHxD9LSCCOkU/ixzj/GGqVAD5ZX4xOi3GJPfpSLMQnJ6D92jdCEYIKSsovXiNPjJntBUvCdtTur1jxghTFplDRWpBqADJYMQcOzquJ1XpW6cKaRS6NgylaOeH1wgjkw4B4Y70qQYD7cM7ylXID3zmyEj4wgh4v6LHcijCJOxELxbwCt/DPkBqBF5cjG2qhbZXjQ0lBCMg+n1URiPlA4+hFeiYMky+f8bkgzbmsJFXT0GHyXlEa3ytpmTBPWHgw0v607OnyEOvmLdfLv556SyLPCIDRBbOVEMqQESRzxIxw0hLrBxLqhGLVYf2A7QdIjHMpSXtkshBIrU6CDLnIjZTqnNA/pj0fzhz31BSy1UuGcRDoYAoGJ7IDMcXXErIBvN9OW4ipolgHCQ6mcbmxJbq4bPMP/nO6ZMOkomvnTJB3jRtmCnbzPWi33KSAwoXyh0OMqLu3zh14kHjxedOGiczVS6YT4qjJaRCMr4wZxaDkZS8arIs7J0DrFCZwOkQBaMUucBAxIFSF3FgAG2N9PCos+NI6KjZoxgnttf2dqWdm1yorDPdIHHZK44PI5eiTGFf/k+UpkGFb4++nlhAiqwajN9i3Vp0nAww55m2zhhC5fLvRWSCjb9Jd6UvYVxnfyc54CCkQA0GPXLx78tn75eJe6+aI19YMM5kAN1pfHb6QVFfK56n9wq5oG9NTKPlfu/R9h2cd8gWTgOc1eghtL3EtrlZ+9REA+1w4bvaa+scVoP+L7EXb288Cm0do3JnO1U/MfiCoz7IIEEHxlPGIow7HDBR2I1xJFHW6WPoV3C0MnWIojFRufjaqRPMqY8OxSNZP07vxI3FbgTDjVRKJhIT+kcJRPCiG8JJJ0YEjAqLsQp2MUWQ9Ds6BaIkweMbBcGPdn4YO3yGDobOL7EQLoo1HWB7iuWxwsB7uPMZ6KCjkZVE+J6ooYuHjOvA63jBEiGNJBrZAf2IDuyx5k8nRu4+HrnohiFPTj570ZHiYXe6HiaeE7WiPRJBoS2ghAWZADzIeDHxVmIgBlCkua84RKIGYYDWwetRI5LWw/fieGEgTYx2A023vbZ1TOgxcJyRQzkk7MsxtCdHvJYoM+irNFk+095PkF2QmIbKdUCe6CsojICDKioTzD25ZEy88JT+gKcVJQeu9+TcLCvkQV+/W/sz2nGQCTYcBiiBtBP2JZJIdBl5wNCkfdMWEp2EgJGUKC/he+mPqZad6HgBDEh+rzPhZ45E0vh9FNlEueAv5DmxujlNlnNhnEs8ds6V8YJrfHAfERtncDJSxC0qE2HDcUX/g9MJ2XOSwwkFWZZZhO5CFhH3N7Rd+iiKG2E0cUuYtkOafQCjnteRi476SdpIaCbsj8FIW+D19to+7Yd06s6EX7H22N4BdgDOf+pwJEIbb9JjRD8M79JeuW78BMfeng5I/xCK4QToU/gcEUPGbcaHqEwwLYK599wLUs+5dk7vxI3FboSiK+RxkzbB/LjYJOS223+3lpkwMy+PSAteoDCoIex4fyl/nwhzPygEEMALhlKA8oDxGc3DBzpA0jO7O4KG55eqWwcf3QHoeDAegPMnWkSfzet4FxPh+5jrEv0+urzQ0TOQkE5CykR0oxIoxR+YD0pVvERvm9M1nKdGOmljpNmR2kKBi/bkgkgI95x9UQJoEjVNOkDqQMfreDsT21CsKmqqtvUDcsGQWRc3IIkKIGOJIBuxttVRq+x6cFhwDJxbIsynRWaix8f5MPjzGQb1RPC0Jxa4weBEiUBxIs2Q+TKJcsFGMZC/rN1lc8KcrocMB5RcoiLMxSXlsT2ZIMJGuibzuJmLhbMLJS+kdKKsYcwkQjQeeYmCIYjCuT9aFu9zD0DFw5R222MyQS6mqmEcnbsMmAEcd5j7GwNH4z47F8Y5PpsIl4fF3KPjIyOFOWR0I8rankzcodf+jyoXzCM73Cwa5+ihOdK+mbuKEU9WSHsywfjBEjM43SkCE81EadK2wG1GLmL9ZPyNCLSfECnG8UAqPm2B16P6VQDZI+uru0Hnac84Q38kQEGhqXC62JQq6nb+ZPS0pwMi54l1G8IYQ1ZXezLBNAaKBqFD3as6bmIqudN7aNvSnaSAx5eOjqIspL0wcZmJ+YkbefSsX4WhQmEDqocSXcRjSgljhBUlAk9yFDoJBkrSxQKkqeF5pkPAwCQNk44jQLorxXZQHLoLPF4cEteGSdtRBYABHqUJJRcFxhRb7bQxKvAeokiROsW1DWeFZ5HzZG4bzwN4jhk8uH7MSfvB0q3yrodXH7R9dvEG+aG+ziRtqvUxSDhdB/edAZv5E7lpA2zwp4BEbGsrG8y9IgpISurV44fY/d2kMkEUmUgMRqRF3+O3HYMJJwlFWQaqMhjA+8p8J5wvtJPESnYolJYWq/IUaUJJgzZK1AjlhPkh0TmawPEhE8g0ijCKKoo+ig3nRHVU5tsmKg4UzWKZmSjIRKV+nnmKi3dXyIceWyvvfuSATLz/0TVWXOcny7bZOl2P7Tx4npvTNYzLjq17SVug3WOQtCcTf9DXmfeEA4QqhUw94F6ipGE0MiagDCa2BVLrqIYYhTl/jBnIEO9np6aYDAEPGJ3IanuOymSA7JNdQAEbqv5i7EaPD2cpc/2RW4xe5AKjL6SrIi/0EVHoH6gGmWgc40DhOrLhwGL9xehY8eHH1snXnt0k17+4zcbs6HxLp2ugv8chwrxDsrKQi/Zk4gbd6KtWldeYg57CdowPtBUyt+gn6T/p+6PGEMYREUuci8HBggxRT4G2x3eMzEq3xygUGItGL5MJDnBSqzkndCic/4myPkyPmUI/OEQ4n+ZWIqHNlq5Nm2ceMNcj6Ev83+a/q+6FcygKwQccrRsr66040Pt0fAgy8R4dN/5Xxw+KpCEXFDhkqTGnd+LGYjfBOi+keCGIVEEN1R7bgzmEGDR0BLE1DbPNUAzLO1AMhIpWUdgHoaeschSUQQpeoHCSJhAtgsBA+aapsfVvuotdapBxfKRQMYczqhjjsfvfOaNk7pCBNgcSI5HOjnmWpPFS0p9iQShVYaBnTsJ4HUxY04v5WwGKNVBiG2PzvJH5bZQGeN2UobZ47P1Xz5O3TR9+0LVyOh/WUCP9lIGMtk5KUWL6cBSKwzAvkftCW2fw5zUKIKHgEm2kzD2RZcCoYvA/Y3iOKb4BFIab1xZLSW2T/vZAW4omCql/zAGjLR5oQckDx8ZTuyrMQYRRjNxGoW3Tn2AYoAjfu3WPLZaOQkORg6F6fSiJTuGNKGePzFXZOLjfYDF3KksyF4ViEPQf4foBSjRFVu67aq7ccvFMW0rA6XpMmVUlF6MHxyIFudqD+UZ48RkXKI+fHzfkavVvinVlqoKIgp24JiV9Jm08Cm2ByrcYqBiSKJ8hSkebQFE+WccZ2kR3wLER2abfZ1xk6YoQKbTjU9m/aHSBHTdz1qm2Sl9CRgLOV873vbNG2f4BCklReTgRsldIfSfd7oLReW2WJ8HQ+MKC8fLwK06Uv108y37T6VrIkOK+0y9TEOmxovbXhMaBhqHIfUdfoJ8kZZK2TK0HHCmMCx9WfSNqDNGevjB/vPWBUUcz8kAfCchR4np9zJ+8bOzg+F/JBf3uHxtKZGdNg7b/NG3jOVZhPgrV5qmFge7DAvs4NqhqjJOJ4MGFKjOk9lJQDbjOjC8UEUIHi4KTFYfStLxM+fqpE6zIUADDmyDGHy+YYXLx8XljvMBNL8aNxW6C+Yp0WiiCrH/GhPqOwMNLGiTeIgQWjyieNASdwRIP2EfmjJFbL5klPzxzsvzmvOnyrdMmxlJcE3LPGWCpbEX6xAdmjZbrF06Vn509VX517jS56fwZpqh3Z1rREzsrrQoskUPy3793xiQ7vp8snCI/1UdKPeNJpqLXPaoUAwoA1R5Zo5IF27968nj5afy8fq/n9PG5o83LHIWUXJQhUiTg4/PGyj8unWWf+75ewz9feIItwIyShmFJtbtD3SPn2GFwokgEEQLmKlIo4FBQKRhjEccAxj6GJuk1KMzPl1ZZ1OwvF86wtk1buPGcafLt0yeaUyTiN7AoHBXslpRUmry8QQ2gOy6fLT/QdsBnKV5x1fjBbeYEJguKNv1g2XarTopizGLLN18009rp9SoXyDrVXDEkcCDhWUYmcDBdv3ybKUgoNF8+eYL8/JyYXGDsYTRwLaLwOZYJuXfbHpmgCtRfLz5BZWi6ysUU+60fnzVFDcRhVtyGBZVZY8zpWoic07ZR2ohqhOkH7cHrtIOw+PjlqrSyJhwFPH724g4bS/j7/06daG2bNnSbjhuv13tK24rCd/E9VEol4v997YsZW2g/v9B2dJO2C5x6UUU6maDkU9WVYyRy/t3TJ8kNemwU1qCd0m6p5ooCzX4sLcLceaKyVGLluK/Q68O4yfhykyq1FHAiKlTfQvGPA1eZ3/rSM5vM6GAZAAo/8f38FrLxTZVBCnzQR+C8PJrlp5wjgz6cyBn3kWVk6L87gjtJRd0d1Y3mXHzNpEIbb0jbprItt/qiMQXy2/NmWNtmQ/c4Z2TufgdJgHvMeoX0xzjgvrRgvMkFutfNF51gRifLWHQXFG9iCk1xbZP18d9QI+5nqtcwnv1B2/hH544x/RNn+y9XFOm1q7P+gaI8rK84SK/LJ1Qf+qWOl8jRr89j3JxkBiEGZhQq3P9zY6kFOy4dM9j0JutXVC6u18/+Uq8jTlvGJIqpoXc5vRM3FrsBPJ6k75AeiSKI0YLx1xF4fpYUV1r1NtLzLPqin8ejw7ppVDXE+MP7g9LIUhJAWkF108EKLtXrME4xfvC4MTCioKNAUG4Z4/Noq9t1BniqSLsllYe0IVICGYSZKE1ECIOZRZCZM8VaQQHmVrFQ+eNqDJOmy4K0fIb0qRpVnighHe3oeF5US2e5Q+7aUqoGQ4tVVWMx92v1GtLJEp1CUbrhpR2yTa81v+10HUSzKCpEWg9VDF+uohxRNJRfBiCiG2ZopqfagEfaMOsb4nW+SNs2bYgIPBGHdaScapsIxNI8W6ycOLLBfUYuiPwjF0SncRQQqeuOFsDxkEnAGpEMzqjmC1WJoZ2y3hcOnm36PkowCyITjYVS7VuYN/K1Zzeb3KPYIxdsRFCfVJng9UQwnIle3betzPoook6k+RHVPGVYthnXf11bLP/aVNKmCrPT+dBu52vfhJLKcj44GA8F/R1LQDToI/0fDkZkBSOG+0YfS9EJ7iltiAqFpNi31xaIxtDX3q9tAblEPq/S9nPWiDxLTcOpk1iFNFlgND++a6/8WMeKh7WfxsnEunlE3i8bWyCTc7KssvGvVhbJndrH4zRBfom2M4b8ee0uS+8m+sJYwWcxDMj0ofIw42OAqD7OkZ/qby0rrTYnIkajFfFQGaTIFteCsYvvPdR47hw79Ev0z+hSpAZzT+mfDwV1CV4qqzYHGVkTpCezNBOOeHQH7jFLAV05bsj+6Tj0gUxxiUJKMm3hNtWhcEpyLPSpHA86BGs2ssxEd8F8c5Z3+Z22ccZRoqaMjRwfbTxXzx+d8YtPb7S2jlwwzx99ijEG4xkDnLmd6EMU/wMcJolTcRhrWIuSuaFMcZqqckABNOSCNHiW8QlzSVnGpr2CQE7vYMBXlPjzPg0dDYuVJnYMXQGDLJOPy7WTw7ihs0IB6wg6J95n0vGGynor/U8lQwYnvKooEXiGWGcLpZL15R7SwROHLymYGEYIKx5nFM8KNcJYJB9vj0Vw9DN4TFHO+TxpbSjQRGie0OM7FHic+U4UjSdU+cR7FM4lpV9/O1fSd0gRCd7uAEo415sO7SmMOf1NzoU0VNY4QqHHIMYDxnfgyWM9ub+qIYBCG13TB8OSa8D8T9J2d6khyLUiuvKo/jbzr0g94nyIRjHAcF2IGjKw765tMuOR68E58FusIXavDhYMGCydAHa+LXq+ejx8F/uitCQL5k+8e+bINhPNuwrOHeMrGWCck77FvcdoY3B/OQOd68C93KjtllLhDEy0GZwwpdYOVC5UVrhfKHkMjszNJTpPu6d9oDwDa23RDzBZn4GRdodcPKZtDlmZqnKBwoxBxnsdwUDLudCWSf3jPJjTEWAOCW2G88SJwXGwLxAlRc5pi8gMczADyA/tnPVEWXMryAULQ6PMc81YzDxcMx6obIfCj0yS4r1drwXf84wa0hR9oL1zrsg+ihSfxBjBQYUCxm9xXdiH30IJYp4iEXn6B64958t94Bwo/sF3JS7J0dWQjpaYWtlVcF3oT5MFbZUUa679f1XhYs3Y0F7aA2cC7zO+0C8v1TZP1gXKcLHeV/rXXdrf0X64T8/o/SINjag8zhXuNXKP05D+mX6V9rDTPlNvbZL+kf6eDBCcjEQWaFO8figGpaaYwojc0Y44JqANUV6fMQoZJRLIcQQyVCZom8g2/RHjFO2cYyQDYY/KLA4d3qe9kpaIs4iMk3tYY1SvQxT6ffoGxhBkib4DOVmkso4izfqsnAvyyZjCb3NNN+t+GMcoyaGPQC44HxwzTG1A3oNjEsOE44ydb0VS9IsA9/LtM0bE/+p6yGpIXKO2q6Ad4fzC2MdwYyyOtpf2wDBs1tvCHLuVe2P3lv6Lvp37SL9HX8yGPDy5s1LIXrpcjUfaJ+0u6EPcR2QC2cBIQp+ijyfba4XqX7Rxotoc0yPanmiTHUEhJrKlaIN8P7IaBR0KZzbtiveRkQD9AjpBGGc4D97lmGi39PkEDXiOrKM7PqW60H9U56QPj46vnD9r9dLnYxTSzyFnyMHDeg7PFVdb+i390OKITsB1QJ64rptML63X826wa0j2DzLBWpfoY8Axx9b6jp0vcpdscHolpg87h6bfvsT1E/ooCNmCW5e08Zz0VFBGETg6CjpBFrkPAy+gkH5s3hh5/6xR5iU98/bnbR8iMBS9oTOj84h6T/m+96ghwpwNBmE8U0wOd3oOpFsuec2CNpPWu4qPPbFefrRsW/yv3gEpNqRJotShCIYWziDF3F7SKU8blmtROiLLS3ZXmfOG4gc4Ahj8UCICzIN6zaSh8q3TJpmMnP/vF5JmQDuHxy/OmSbv1b4rGaCUnXzbkvhfvQPaMCl7gOKYmD1C5OHd2u+fMizHIgVfeXazKYqML1R2ZGxBISZqASpKNl0BWWJ+FgrlD5ZuM8XQ6RmQabTompPif3U9b35gpRX/6U2gD9H34xyn+jNGWbCfaN/M+73rijnmgCHbibGQcYTIPPP7cEwn6owsbo8ssXYzU36+88JWyx5zeg6fnz/O1ql1Dh9PQ+2l4G3GqKP4CnP0Eis4MgGffHo6vM1VddIa7wEpYPPPy2abcpVocBBxYeAnjZPo29KS6vg7jtN7YCL93TrAf/LEsQdVNERhplpubP5XPxvAn95VaYblayYPlVsvmSk/OnPy/jTuACnfKNFUfSSKkMxIsuN0BhSaoN9nowpuIowfpKxiRJLpQnVEePWkQnnoFSfavN2RWQcK2ZDOTeGPmfkDLXpGxOL+7W4oOr2LN08bJn9iXrvqQ7R/nPABqqx+fv54K8pCMIFINLBmKXNjkSUK3yXCXHlSOKky+ljRXtmlRqjj9HY8shint0UW8YadPixXfnv+NEuRIPUBA4+J9XR4VAwlTYO0nF++VGQeP5Tc/5kxQj44e7QVr2A+B5ObWXCZ+WKkreAVIy3jJ8u227yOaLTS6X48svjyfOXkCZaqSzELUuOQAVLIiJIwf4v0IFLNiIRQIRGHyrT8LFMY8CSTlkqKJZFJZIm1uZgXCETa/7Jul6XoOD0HjyweGvoNCtPgQEQeiATyCMz7WlCYbelupIa+99G1tnwK0wIopkREPSOln6WUkfJp44X2PxiKyBKVWZnCQXq203PwyOLLwzy+d54wwnQp1i9lSgz9Ps7EWOVslqsRW0KLpR9IUaXtYywy/49xhbTpMJUBRzvVUNGlSPd/9yNrrBJviMg7PQOPLB45PmcxTjLnLHYGDOQ2YX9fP5vYzxqMTC5mviHV84iWrFIDknlJzGXCY4xXgLx0Pkf6BYoxBiWfQVlm/R3mszDoM8mZdCWnZ3E8z1nsLIIEs04nbXyKGntUZOM5y3Og7FMMBuWXOUsM4zhFcLowj5aBnoIVQZZIT0WxJm311g0l5mFG/pyew/E8Z7EzsIJe2oaZc0uaNpFGihwhF0RQmJZAkbDfrdpl83SDassjCjGfYT5xkAscKIwhzJW6Zd1ueXBHuc2ndXoOx/Ocxc6ipqnV+nIcJdbvxx2DOFdIv0YvvGn1LrlL9SEMRabt0PMzVYEK6yOy0qxY4bS4LBFVTNHXmaN6/YvbrQBOdI6h0zPwOYtHjkcW4/S2yCKY92tQhlVlw1DEQExhMolCdJDoCJP8EwvLYFheMLrAOkXWzSFVj04QxZkCBqzTw/Vweh4eWXx58PzO0LZNRVvkgwqHzDNhMj+Fb5ALig4whzcKKapURiRaQlVV5AsogET0hGVnVqgi7T1mz8Mjiy8PrfmycYMtDZtoYlgSAGV2Z22DFZug+nMUou5EWC4dG1uvEKWaMYbxgqI2FBqj6NjLFRdxko9HFg8PjEQcTfNULliTlvbNHEaqCuMkRB8K2SlRqBJN9dPowvfsUxLXvW5X49npmXhk8chxYzFObzQWnb6HG4uO0xY3Fh3nYNxYdJz2cWPxyPECN47jOI7jOI7jOE4b3Fh0HMdxHMdxHMdx2uDGouM4juM4juM4jtMGNxYdx3Ecx3Ecx3GcNrix2EdhmYCCjFQrmELp58LM1Pg7scVoKQNNxbzDgTUaJ+pnZg8eaCXWWfy8K2C5CNbJo9Ilx+44XUF+eoq1ZWQjtGWWHKB6MAs3Z2l7Pxz4zCSVC9orshSrrdr5DMtKsyU+pujxdtVvOH0b2hVVH+l/6eMDtGvkgmVpqKx9OCA/yBGfQ0a6AtZHDcdLRWQqJDtOZ2N9vLb9WdrO6Idpd4wZyAVjCHrR4bY9PoPexWe6iuy0ASYXjBfIBZXwHedwcGOxj0IHxoK0/7h0ltx2yUy5bvqB9Zh+fvZUefDqefLJE8fGXzk0LDXw83OmylPXzpdPnzTW1uHqCkZrZ/rwK06U/1w5V66ZMCT+quN0LqzZt/hV8+Vfl882gxHOHZUnd10xR559zXxt74fXvi8aky83njfNPvfJE8fIgC4amN91wkj579Vz5a8XneCDv9MlsPTMD8+aIo++8kTr4wGH4yfmjZG7tX3/5tzp2t4L7PWXAyX62dcsMLnoqrXOUIh/pMe7SI/366dOkPmFyVmD0+lbMC787vzp8qSOF++bOcqWJWP90k+r7vT0tQtML5qn+tHhgCw9oHrXL/QzXcWZw3NVjifL/TpefOO0CbZGt+McDm4s9mHQK1Eu2aI65oD46zweDqozmOKQ2r+/PR7mx9olW41YFkz99umT5MsnT7AoZ4DvZe07tn78qON0AXSKtDHW2wrtDGU5Rds3bfxwW7h9Rje+C7k4Vt4wZah8V+Ui6tgBvjvIseN0FbG2HOvjA6Ht4Qg53CbObrF+vP8xKyCTczPlvbNGyV8ummmRkigkBcTGsWMbkxynI6J9vP5nyhAPsfEi9t7htj3aaWeMFXweh/2nTxpnS0Sw3nCA8Swqxy4XzuHixqLThhte2iGfWbxBbt2Q/EVlRw9Kl3O1c3vFhCEWsRyUeqCJltY3yaf0uL7w9CZ5cldF/FXH6XqWlVbLl5/eKB96bK1srKyLv5pcrhg3RC4dO1gm5hysFN+zZY987qmN8u3nt0iLr5rrJAkWLr9tQ7F86ZlN8uNl2+T5kqr4O8ljSm6WXDuxUC4anS+D0g6k+7Fe8i9XFMknn9wgf1izS9ZXdI/MOn2PbdX1qjsVy0ceXyc/Xb5dNiR5vEgb0E8uHpMvV48fLAuG5hw0ZWdlWY38SuXis0+pXKzeJZWNLfF3HOfQuLHotOHfm0rlt6t2ymNFe+OvJA86tgmqDI9RozFDO72o76uisVl+vbJIblq90zo9x0kWGIgonTcs3yG7ahvjryYX5sYwLyZ1wMHd9jPFlfI7lddb1hWbAu84yYCW9tjOClM6/7mxpFsMMtLoSBXPTR8g0UyY8oZmuXvzHrlRx4v7t5XJjpqG+DuO07WU1DXJItWdfv7SDtOldtYkd7wgYsjcR1KxydSKsqWqXu7essf0u/+qXNQ2u7HoHB5uLPYwMlQRJLrGxHxSMHPaKQCQEyYp6z4UG2BSNR0E+/I6gyefZ24IE68psIGiyecOJ8UBY400hsQCN4O048GI4/v4XjaK4wzNTIulYCTAS6RVDFcFl3QhJlVzTGwcO58dgfIb/zCKMPMS89NTtWH2044uRabkZcq4eKdHgRu+g8IIBbpPFOZgci049+hv8LvR4j1AgQWuMefIezznWMI58RvtdbRO94FckGbGvaE9JDZjmhD3dYreb9oAbTIwJO6AoN2GtoF8sB/t+XDuM7IV2la0wA3yRJED2gvtKXw3x8H3djRPMSslJue0u6iscn58lmOmvSPbyA/v8RlSmwr1Pdo214H3OVfOjc9Ff41UKOQiFNmJtm/6A+SZfaKM1e/kmMboteYYoscX5ILvTPyc0z1QTIN2xz2iMFN78Dr70AaQI+CR/pY2w32lXYS2yz2n7z+cQmUjBqbJdN2fohzRYjXIBX0032/tR7cgu7kdHCfk6Xu0wegYxsax0/ai/T7yg/wyfvCPKCP7IQ/IBb+FnLBPYpERjEwbkyK/wbjC+INcRKEoCOfHNeI95DE6zvCbjD2Jv+F0H7Rt7hH6A313Yn9Ft8x4glzQ1yEP7EGb514iA9G2QdulvfC9L0cocEN7YdyhAGCA3w3fT/tmH9oPbT4zsl8U2jcyTJ/NvkFWeeQckYswJvHIcbI/qaa0SeZRcvxp+jdjEnLB9yAXjCcBxhvadrTP55HfQE9DpgJcTn6D3+Ich6nMcRyhL+FzfA/nStqr0/vpt0+JP+/TbKtukAW3LrH0le5k7uBB8qE5o+Q1k4bK4t2V8rPlO+SuzaXxd2NcMW6wvGfmSDljeK6lO/x42XbZrcd91og8e32aDpoIfXpKP2ls2SflDU12ft98bot5gisbm60Dff2UofKxuWMsGkHU5DvPb7Xv/8+Vc2TekGyLVHzsiXX2GlwwOl/eNHWYXDCqQIZmpUpTa6tsrqyXnbWNNoieMjTHPLlEOZ4rqbKOiOP4wKxRcu6o/NhgnxFTFEh/IEJz79Yy+dGybfq8Qb64YLwe0zCZqh0QnXurtswWPbY79fz53nV762TxtfOlprlFvrFks/1OgLz8104eKhePKdCON2bk8ht4u7lGP1y6zV4Drtv/zBghl40tsN9mQvrJQ7NNIUDJIYL5bHGV/OKlHXLP1j3xT/UM6LiXvGZBG4Wmq/jYE+vtGnU3VE/87Enj5PzRefJo0V55ywOrpKGlNf5ubJCkkMBPF05R5S7Foh3/99xme+9/54zWez3YBrBh2m6Bz+6ubZInVB5uXrdbHthevj8q97Zpw+XGc6fJVpWZ1/53hbygbZm06O+fMdkG2HP/9YI8pbIJKMgXqly8dfpwa0MoofXNrdbuXtxTLScVZtug+q9NpXYtm2nUyinDcuTdJ4yQU4flmtKSobLKe9VNLbJdf/ePKo94gFEU3jx1uHxU5dQUfR13OU4iJ4/t3CufWbzR2v17Thgpu7QPOOMfz6lcxn6DNoIcf+PUCTpwH1Dmg1x8+ZlN2s4rpUp/M/DXi2fatSal8Bk9x3drfxKOr7y+WZ7W/b//wjZZWlp10Oe6m1+cM03eq8eaDJZo33DybUvif3UvtPl7rpyrilw/+fDj6+VP2m4SoT1/6/RJ1m4uuXOZrCirkVmqCL56YqFcNX6IGV20Ffrr+uZ9lkZ3//Yy+aPK0Aul1fYd9Iu3Xzbb+tk/r90lH1i01l777hmTTDa2VtVbJOW2+NQF5OCV+vr7tO9HyaXtVmibZfrARh0zPjZvjI1JH3t8nfxjY+wz9PmvnVyo31coCwoHmdGXETdYi1VWn9c2d0c86wV+rTLK/F2bL6l/0+4ZU365InYc3zl9oizUMTFEUoj2BD44e5S8wcaaLMlJj/WlyB39wN903HtoR7m9BpfreEvxkZN1fLvmnuVynso74/M4HWfoMorrmmyM/uv64m7JxumIhSNyZdE1J8X/6nre/MBK+cva3fG/upcvnzze+k36068+u0ke1Psa7a8w4H593nQrUMN76D6rymtMnt6uugHtHN0FBzuUad+3Wdv4fdvK5CvabwauURmibZyk/ez3lm6VP+n5M3XmbdNGyLu0f0ff+rrqKowzwDj1PZUZ6jLgmKB/L1P9bPGuShmj+hHOFaY8XHjHUtsf2O+C0QXaF4+w8QdnOqmmDSqrtPe1FbWm2yFbp6tu8/vzp5uuhfFH+0Tu9+hvnK5jA/rDu3WsYPzhvJnWszueKYP+80499yvGDzaDEqOX3yAqf9eWUvn9ql2yUq8RIM+MOx+aPVpK6hvt+Bn/2HLTUqVRdUPk+5Ede+36M4WoJ8Fczm+cOjH+l3M4vLzr0EkqKHEIJQYRAxkenURGaYeBYO9Vo+bGFUWq1NbboP+Zk8bKadoJYGCtUKF+WpU98uXpGGebETraOqmXI0zYjjqWGTCpuni5Kt0DtTMkDXRVea15m1CI+f7EKAoKw3dUSaFT4Tzo2DgmNtIfUFKu1I4ppggPkA167i/tqdHOqdEU551qQC5ThXu5vkaHxren6W+wBU8hRi/K0BfU0ERpoROls31md5V1UHR675wxUq5XIyJ43zhMFCNSXjFOGVT5PjrCddrxEsE5c3iOKiLDzTB3uh/kgntDuwhyQTsIMKhfrfefAf4lbTNL1KjBC/x/p02Ud+gASGQDxSG0v52qfGLEUbEUx4HaQoeE9kH6JwNwaHtEOVBGP6cDz1k6SKuNaO139d5aOw7mF+J5TQQlmmp5l48bIjmpKbp/TFbX6OdQeKflZ5msIVcYdrxOqmm1ygyGKIMwRt5Sbec1ek4o7ana7pHZAArGW1RZun7h5P3f85LKLLJEv8F84B+cOVmunVR4UEQI2cf7fJbKBMoOrCivtutPcQSq6aGE4413up+aplZzzGEu4czjvkUhgoACjKywX522nwWq0P2PGlkYWsgRyiBtiQ1nKcbd6ycPkwtVQaUdHYoDY8UBuYC3ap+MImnjlBqJ9MnF2h/PVaX6jVOHxfc6AGMFhuL/6meQJZTclTq+0I8zziB3yBr9Nc4MlFWMTvqEWpUB9l+ucv9cSaXJB0dCH8+xIR/hyPj7J2dNkQ/Mih0bSjS/wbmz35Xa3yPPUbnQj+jn+tv48ZWTJ9gYWKey+FxJtY2vRENfpfu/Vg3Iydo3Od0P9xQdhcg31a0Tnavc00m5GdLU0irrtH/dXlNvugJGJm2AiBwpm3wPbRedisgz8wBxNHDPO4LWZn2ythnkI7Q9Im30uXz/SJU7ZA2HIo4/9DLGqMTlZ/idV04cYtW0GUsq4v04bQ+nDs7PmPN7uOkxBAJwbuLoRNYxRGP7V0lDy75YgRs9eaKMPIZjY87vN3WsxOnJ2EUQgDEJ+RqckWKOlS+ePM4c7AGuId8xd3C2VdbHGYvuxu+VqawT0UQnQ+/ECHZ6N24s9jAwEjdpJ4WXEwWYED/CGxiRlW7KMoYOgyL75qWlWuh/nCpwvPb9pVut2MV3XthqEbjv6iODG+XDJ2iHeDScNypPTlElgygi34sn61u6/WDZNrlva5kN3qHjATpnjhNFlY7zHt2HSA+fZfvi05vM04uyQidHp/PkrkrLo8doa9bOmQ6P+Ynk/W+pan/OCYrtq3Wgnjt4oHm3b7KIEue+Rb767GbznnEdL9EBHgVgaCQlFcUBT93928otesb5cK3wjtbrIIJCTSqG0/3gXMCzW6Ttj9ThS3TQylcZCKDI4Q1GqcTBsFQ3nCQomMgMEQM8vKH94R2+X1+jbRLNZ7COGk2HA0o1bY9Bkt8kovGNuFwQtUMJPzAcHwCFEnnFeOWY2J9j+saSLZYlgFOFaN6pw3IsuoJXmnZZpN9HuyTix37IBYpGe+A5xnhGISYKSIaCyaz2C9e/uF2eU2MT5ejaiUPlHJXtKHiUkQ36FvqRbz+/1WTplvW7zVg/URX+kwoHtUnvdpIPWRD/2bLHoiYz80mlPLi/ml+YI/O13+b9/2wpM2WS6AJjAQrjH9bstkgx95j28RNtG89q+6L/PmtkrvWBR4KKkzlg2GgfODW+qLJAu2Ms+vemknbnSZHqifyiUKNA//ylovgYtsVk5PaNJbJHFVBSXlFAcRL+Z+seixriREGZ/52OFbTzxR0UP8MgRV7JPiEV9hEdf2jX39bf4Nx/r59nbvIc7fMxdhMNAs4NRy1RR2T227p9XWX2cZVP+hFkAmet0/28oH0e7ShV7wvOkmgKJe2SiOBIHRfQZzAKkQXGACLuONNob7QJ2h/6xE9VLjAax2dn2hIxwfF8uNCWGGcwFDP0s39Zt1u+Ym0vNh4RscSpgh4UhbROItpkZGHwUUjK+mQ9JiKZf1T55ROnDMuVabovY8RNa3ZZlBQ5Qye6Y3Op/ETHC76/PXCqnqYG59k6VuLwIQrPeMNx8UjBQ5yYZK4xnnLdouSq7DbquERmARHab6qex2eeUDlEd0Wuh0Z0WKd34sZiD4QUOcL6KIYM7CijAQZvvMXsg9KI8sbEfjqHv60vlt+roJNWhyLJBH/Scf66rtgmXWNYJXrYXg46L/Lp6ejwttFh/WZlkfxTB31S6/6snRVponjJQhof0IFx/BwDqVFsHAt/s5Fmx/Hr4csQ7bzpIxmoMRR3aofH4L+3oUkeK6qwjp9U2kRQGLg2VPyiM3tox177Hc79Lv2NW/Q6cC1IGxulndVrVFGIzjmgciTKNp3pX3U/zucfeox0lkU1jeblG5zhnVxPgcEfRRZjBoWPyCDQpqkQSvoyHs11FXUWicbxgkOCe4siyGNofzx/UpU87j+K4xyVq6xI5d3Dgcj4QlWokcHbN5Za20PJoB39RtsQSiQpOongrb1LldybdB+Oi/05JoqEsG2qrJd9+g+DcXBmikUvMAg4Vtr5RlUAcKqgvHRUoIDoCxuG529WUvwm1iew8fzX1sYbzJA4PUHBxVDEs4yMcp1I/fu79i0UCiHVNV2v/xTtlwYnzBt2kg/G30Pby+1eMjUgMRJINASnwI7qBnlY98NoRDYW7dyrSqW2P20HpIHSh8faR4kpecgO7Q9n2pGAyxDZnJE30ByXtJ8/x/tk2hCOOJxziRAxJ4J+6wYdw1bv2j+G0Y/TJ9P2GB8Yg1D+6QNo/8tKa2yqRavKC7KOc5AoeHswB/d1k4fJyIFpFn3h2EKfwG/9SccyHEiNKmM4MHEURh1IzNhBdrlWt+i5MG5QDZa0eFJRCzPTbI6c0/3Q79EOyC5CJkjLDHNKccBdM3GIOShwZiwvq7H+liyNf2s//gftk2mn9MW0P9oGRhmRNozO0SoX9JFHQnD+YTyt1XZOm472rRTuQ/eh7UVhbGHco23iCGdfjoe29/f1JdZHVzY1W9tmPGSMwLnI+WPA8TdR80MVs0G3w6lPyvdzqivxW7fpuYcxid9FV0tXg5r9TtIxIwp6Htk0yC19CeMZ53bPljLT65C3IzWunZ6HG4s9EISaDgEFAGMRYyhAdI9oAR4jFEiMxu14k1RQP/HkevPoZA6ITcJnoKdIznhVpOmCDu6GDg8ihqfqb47QDpeOh1RRHoNdyLESBV1dXmvGVwClhI6YY8KzTIdFVI8JzygvpFSgpGMUHi0YfqRdMaATeWSeVaKiQMeJB5oUWSasRyebk4KCAkLHSsoG0PFhENMB85lET5/TfWD0o5gxgKLIESlg0KaYAF5h5iouI41Z2yOgFH/h6Y3y4cfXWRonaaPIBe2PqCAVFFEyjxbmpqAg4ohBwUY5jvKUvobXOhFk++NPrDfPLYZuKLJEFgHzbVFcjv6oYuCAwTm0fm+dKT7ROSPI78362lp9D3ngHBJh8EehieouyEbwTnPcRHGd7oV7wpwl7hfRLdoRiiOQfTJKFTXuEu+zH+MFUXYyO1iCiOg3bYB+mfbHo93yY2iAjDko4syNRNmOwjjB/D6cHtFyCfTBRMvf9+haM8Y4L5R6ZJXxjmUxjlUmsrTN4lykXyeS8+iOg+cXMha8UFJtYyvF1Ui5RtENIAso02GOF3AKO6rrpZrxQoeKaATL6V7oj0mJRM84b1S+pX4SOSPKRzvAmMLBjN5AH46j4qPaL//0xR1m0NH+0KEwNGnPx6IJ4HA/sXCQZUwxt5AaEwHGM/QUnCVVjQdH/xi3frB0m3zmqQ1mhKGPMN4hF2RlcUztZa8cCThfpuVl6jVoNJ0SvZNrA8gp7Z1oOjI6UcdN5iVGKdMxgawfHLUhYICznfMBW0/V1ahej/dsPRAMF1JktqoBhEGEIAeo+EZnh3AyeRphjsJ8QrxmzBf56ikT5JfnTLWJ3KTvYPgdKXSupPuhlNMZtFcAiFSomLHY8XBORJSJ48yrZII3iygzZwaFvzNgoKezSwTPOwNBVWNsXovTu8EJwXxDFMg5BaRCptncCOZroawRXcChkQhOl1dNGmLFbr6v7e8PF8ywif5UCj1aUM7ZmIuVKIewSOWTKGFHYHBRHAc5oLjTL86ZZnNrSWU70gyA9iDNtai24yUDiNbXtfScIjXO0UG/trq8xvphFLm3TBtmKZMUsCG6vDc+7zWRgSn9VYHNtjlR9Ms/O3uqtr+pNsf3WPplUvqYH0XaKFsUIpak/pXohuLcHkRhmPbwzhNGyA/PnCy/0fHr8/PH29ysY4VfpFgJii/XKxHGC1Ldnd4PGSZkJqG7UBsBA4t+lUjjhOxMu8/0kYmgX5H6yVz2r586UX6qMvH3i2fa30cLx4D+1aLjBBHxPdoGEylVPQVHXnsQ1cNwpfjTx+eNUR1qsvzu/OnyNz2u6DSlY4HoOI51ZDQKcrq0pNrmPzp9FzcWeyh4aEhLwECjY2PwZK4QkQcGYDzF0bRPCtz8URXg+6+aJ5+dP07eNn24FShAqW7VDgolGw/WsdLeN5A6UdF0INoYQOH49mmTrFof1SXfP2uUTYSmI85Wo7ZBjWLmUnYGh4rG4ME+9liN0xPYVFlnFWppykQTMfaILBMRwTBjIGbQA9ofxuFtl86U36rCSTVVir6cMSLXSn1j4LWnLBw57bctnBftff/5o/JNCb7zitlWgOeDKhcUUiJlMH3AAFNmO0dWD93q7b1j/xmnm+EWkjpGaiWpaCwHQLSB1GocfcxPjVapZBoDCue/LpstvzhnqhV0efWkoZaOhjKNwnosGR8xYm2vvW9h3KpvbTUZDiDDpK8yhv3pwhlWGI3qtlSrJFqKkk0hm86AUzvU6XmB+OMDonekcOJYZ64p01Zo+zjoMIBIsySKDGRJEEHEYXLbpbOsqvb7Zo20Ktp8Bqc50e6jJRpYs3bfThtrUJlIdDpiuFKB/u+XzDKD9QsqqxSbYeoAkU8yoKJ64LHS0Xdx5i4WfRs3FnswT++uEiq+MVjiUaKqKPnypFo+X1wV3ytWIhsjjEIVDLrMd2SyMRO02X62fLtFBDvy5B4KOo/qxpihiSKRWK0Lwlo+0VQDjpliAkQ56WxJZSCt7ZvPx46J7eGi8v3pn8fK8Kz0g+aXBKhsSodLzjzedqd3Q9rcIzvKLV2HVKILx+TbXBBS6V4qq7Y5gpTtJgWTCr0Uqjh9WK45NCiyZAUznt0s33hus/x3W3mbyMeRQNo18kG7a6+aKsUSEr2+GLUoxTh3iKCQqseyMMgDBUB+rrKKcsM8rGOFdbsONa+Qa+SppMcHyMWW6gYzzoii40ShUAvNktdJLQuQkkflTpwT26vrrV+mKNrXVC4YK0j1PhYnSrPKH6lnKOdsiZBKR7uMpviTQYNxSIYAURjGsF+uKJIvP7PZZIMUwY7mIh4J/CIFzzCq25tHxXjRWdkuTvdCKiV6D1VRSQ9maS+MP4o72VxfHUeCXJBuzDJBOO1wJBKVJFWf4kroLBRoejiynMqRgu7FGITROTGXtNa2coGsJGaUXDS6wCoAU8GYz5NCToEyxi+O60eHKFxzpFCoBkd+Yio12TOTcmKpuE7fxTWFHsyy0ipLL2Iy/9lqEF6iinHGgH6mYDIpO0DlQyqxUaaZ9d9+q8rnDct3WHVGJkSzVhae5qO52Xi6SGGiQ2J9H+YaYgjS6QHKJrnzNhckYo3xGh48vHXMHWBCNgYsx8Vkbjo91rmLKgxHCgYDCgTRSX4HJSmxQiPVAZnzybHtrGmUBj0fp/fCEhAM5BR94V4TpWNtqdg83z37jb88bauUIsdgJCXugW1l8kttf1RLRAlFLohSRtvskYKnuaqpWQoz0izNOlo8CVDKE5fOQIlnviXzyjgPikVxTMgG846ZQ8MxHcNhGSj7RIeYr3zKsOyDlBCUZBQnlr8wZ1AnRWyc7oV1aJk/zpxBHHW0NebvJhpZ9NUYiqRi3rhy5/5+GacFc5bo84+ll0RBZw1e2j7z/qIw9x1lnXlkUdnD4XJRvMokUzAYuzgmjFcqYjMGHY2zMwrGA2nZGKMo31QkjoJMMpeZ8Q3Zpo9hqQKn91Ki4wFzYClec/qIHKvSO1zbIDJB/0vGFXDvrxg3xPSWZ0uqrC+m/gPtj/WcKbLH2HO0kGJK0TD0HaKCZIgFRx26FG0OHQbHdhTa6Uk6ttBHU/TpVzp23aDHxCPnRfT0WOUC/QxdjCkdGNNci6CXIaPIJtk4RDJL9ffam4fvHP+4sdiDoXBNMNRQANhQhpl8HfUUkwuPJ5cBjsXt6UDodChYQLVDW9dnYNpRRREoiEDBF46DQZY5MXjfUATo3FDIqZBFumt0nUU6GSIbaL3MjSKySHQyFLe5RBUDDNxEzzO/R7SRqnqcF+sk0Xm1VzgAZQcvOCX+2YfKlCjoHFf4Hda043iDJ5G0Xqd3w8C/qKhCDZ2Y4kslVGSFwZT5qUBlN8qNkzuDIcn8FOZihIIZKKyk3R3LfA/STFm3kEgFqUEXaNtD5mh/KOOUIk9cjzBfj4ljI/ppBaNUgQiyyrkgX8xRbm9+MfKDfo0HGOWCCEhHzpaNqgiRYsX3snYdMspxsVHQ4H/njDL5wLhAaXJ6P6wtSgEW+sL3zhplKagUyKD/jkKbQUENa2/SgpAD2gNzHHHyEXU+GjAzn9VjQAHFKcI8Lyrn0u4wxC7Vfp/F0mNTAw5AM6aPp43TbpFj5ILP4QRkviIylQjODpRlnKEU/kAu2sswgQqVtwe2ldvSCET3WRuRc6b4CL9DyquV+VelGIX4Ue1jouOs0/tA77hlXbHdx8na75OiTUozBWWiU2CQB9Zopn/dUlWn7bfeXmO8YDmXN00Zbu3jaN0oOB4obMNvsswFThRb7ky/nzUIr1JDlWUyEiOLyCp6VbW2WdYRJe0UuWYMYymLD88Z3aa9c4ToUSSnMI7QHyAXHY0VVHmlyA9BCaY7YcxyzsgF1X2ZQ3/uyHwb517U8Y55oE7fw43FHg5Vpei48MSykaKGkRQFRRmDjLS2d5wwwooU/PisKVYYgLlanzhxrBpvpM0dXUcHTHxGsWBR5a+dMkFu0N+gIAdzTD45b4wt+hwFTxieNAZzUos+P3+c/PTsqfKThVPkLxedID/S46PDpCJplI0V9bYYM4YpxXyYW/MmVS4YzNuD36G6JAoAi6B/67SJcoP+DnPCbr5opilNVGHFk8gaRWGOgtN7oVgRJcdxnKBQ4iihGFS0dVeqYkhbqtP2xSL3H5s7Rn6l7ZUFuX8fLwxw1YTB1s6OFpRw1ufECCXC+RWVC37j5+dMlQeunmcDb+LgT9VeFGEGeBY4Rj4pavNL/dyfLzzB2i1p3e0dFjKOHGOUfuPUiWagdpQy98iOvbb+KWl1r5k81PqDG/S4YkVMptgSAqQVEeF/cFtZ/FNOb2ZFeY2tL4ojgmIYRNKoQs2ySlGQH4wyMlJuv3SWtQmKeNBf/vq8aaZYHq1c4MAhKoMTDwPusnEF8k/mgOlvMFbQblE+cWxG5wYSzSQimaPy/Ekdr/5w4Qxts5NNLv5z5Rx545Shkt+OEYgjCAcgzskvzh9vYxPjTXuQqktlSYqxoUC/Uw3Zv2o/gMwxZnzrtEm2gDvXhyUA6EOc3g8aBv0uznLGC6KNd2zec1BGBbISc2iLfHDWaKuxQLugDTK3F6MMx8LRTg8o1u+mOvxiNcwoLPXxeWPtN5gbSaGa754xyZwhpHBHIVDANCAciX+64AQbvzguCrQx1uDYSdTt0LnQvajQO1mNveumD5f/U7nraKwgw4axjDn/OBfR0YIO9TPtFzhOphLhAGK/9RVti2U5xz9uLPZwHt+510om4yliu3VDiTytnU4U1rP589rd5rlK6ddfFhRmm4cUY4sBmHXVSG8jkkFaQ0eD6aFA+fzp8u22rg+pG7Y+2/AcU2xJc6VEeii3DMy1/O2qIrl1fbFVvyPCcpYah2cMy9UOO9Xmnd2pn1mhA3dOaorNLWM+F54zqlmi4NC5kS43f+gge689qGjHWnXMa7lXDVq8Z3ih8QKOy063Tpp1E9+3aI150o+uq3d6EhiAKMVEu4kYEwWwojdoqnGo3PaYyg4LGK/YE29LqqTSNpj3u1UHPtYfvXPLHjOoeJ12eCTUNrXach5vfXCVrc+WrqM21SVJe2Z+CUpzYlRnnR4zxUaYJ8bacqSvstAxXmXGe9ZmZIF0FFsiMVNVhgPIGanURNuR71MT0kujbK2ulz+orH70iQ2ysqxWhmelyql6XCjqRGC4dp96coPJxhbd1+n90H5wJL6k7R2IAvB3RI80WA+UNLbS+kaLUOC0m184yBRpooJvfWCVRZtJ8ybKdqSguMbmzG/WfrxWClX2iFYwHm2viS0SjsMEgzWAwvqBRWvl6d0VFg0h6n/m8DxTUmtVTq5fvsMW3ceG5XtCpgkKLnPJSAdnjjByzDm1B79G9V+W0vnVyiIbl5hrj1OHzyG/rOf4f3rcpId79cfjAxwRd28plW06ThBdpu9jig/TEwK0d5b5oi3hgMAwpI9lDh/Rxr9vKLY1dGlnVFMlEt9e9kdH8EukiX9m8QZL96a949Ch3dHP4+ghCwCDLAop2OxPZDIvfYDtz3JhLImzZm+NjW/IWJYeC/06y0M16Xmx3ihTlXCiMBWDz3R0vBwb6zB+/umNNibiLJmp4xFywfiEYwfnyZee2WRO2gNXzelL9Nvnpb8MhHTBrUvMuOppMPhdMrbAnrNGVWKHApbGpgI+UjsgyizTwZHOyfpqeMxCERrWxGHOIxFKFjFHuaUJ0OEExZbUTfLXSXfFQxzAG0vHRtoEBh8DNx0LRUWsgykYaBVcUU65jnROpPsxb4pICoYcigQevW1VsYX3WQ+OYgd0lig5GH8YhnSKw1TBJRWEZTFI98PL9vopQ63zJ3WC8wjQgZM6QVpHmIiN8UrEkfXkSLMI0KnOLMjSASHLlHIWZ4+uQwfXTCy086Xq7GORa9DdkJK15DULOjQSOpuPPbHeomc9EeafjM+OLcS/RO8vUeNoZ4YnmTZP2uWQDNKw+5nXFeUTxwnzuViInwXESd2mrROtREElUohTBCMUxZZ2j1FHeh4LcaNoAu0TT/Fp2l5pgxiewGeJxKMQMIDThjFe0Y+JdNNWWc6GVG1kFUcQzgzaIcYvRXuANh7a7ol6Htx/DF9kh7aLccl3oRCQnvu39cV2jkB0CGWfOZ0U9EAZBrzjyMVi1vvS88CwDVw+brAdM5F4FI4ovI7yhHwyh4cITaLcdBdkOVAgJRngIDj5tiXxv3oW3CP69HGDMqwd0afTTqLQL5NeRnVtDERAwURJpl0z7/3ycQXa1lOsDdNOaKPM6YopqbXmPKQ1LYxXJEaJXran2hTiANkgpD/TjyKLpP8hZxhhZMEA65PiHESppU9jKsHIrHRL1aYdo9AjSyj3jGE4QhnXMDiRYY6f9EJ+h99AsX2xVI+jotbS9DheDErOI3odYmvLZdnng+HJGIaThXEvOsYytjKOYSA8X1JtdQQYR4HrgkMHpZo0PtY15Zx6AhS+W3TNSfG/up43P7DyoKq7PQX6aMYBHOXUXaAvTuzbAP2EaSzWj+u4QGScSB/6CsYkS8Iw5YH7jKMbHYdUzZNUjnCs0E+jL5BlRboradg7ahpNVwnpzDHjbaCNJ8gFWTHM77WoprZ3CgjWaPvGMA3Qv9P30+/zeeQChym/j+zgaESPQs/ht0h1RVYYQ9AdOV7k4m9qCDIO4FQZoTJGWyelFLkHdDRSY5nuRMop1wOHDuMKzn90tDCNh+OYqn3IPD13xhn6BH47CjJ+teqSXD+mAPWkrC4y3chycA4fNxbj9GRj0XECbiw6TlvcWHScg3Fj0XHax43FI+fw4+iO4ziO4ziO4zhOn8GNRcdxHMdxHMdxHKcNbiw6juM4juM4juM4bXBj0XEcx3Ecx3Ecx2mDG4uO4ziO4ziO4zhOG9xYdBzHcRzHcRzHcdrgxqLjOI7jOI7jOI7TBjcWHcdxHMdxHMdxnDb4ovxxumtR/rz0FPnArFFSkJEqA/r1s9eaWltlQ0Wd3LVlj2zX4zpcJuRkyMVjCmSwftct64olK6W/nDYsR38jVX7+0g6pbW6J73lo3jh1mBTqd7xUViMPbi+Pv5pcMvXYTxySLeeNypOndld2+XFcPWGI5KelyB/W7LK/3zx1uLTKPnmhpEpWldfaa4GTCrPlFeOHyC3ri2VzVZ3UNbfG3+l6+sqi/OfqfT9rRJ4M0XYYqGlqkaeLK+WOTaXxVw6PeUMGyUL9rrKGJrl1fYmMGpQur51UKOtUxh7fWXFYMj8lL1MuGK2ypbL0hzU7j0guO5tTVaavGDdYHtpRLs9r+6xsPDy5PlKGZaXJO2aMkGV7qlUOqiVb29zCkbn23r/1HpTUNVn/8vopw+w1aGxplcUqr7dvLIm/khz6wqL8+TpWXDA63/rFgRH5ZwD/wQtbZXvNkbXJsdkZ8tG5Y+Rh7VufUbkap39fqN9Pf/eQvra3sTm+Z/swWtEXnj0yT8fPerltQ3LveRTGTtpquo4bz+n9eXJXRfydzuccPV/6g921TXL3llK7BtyTLVX18t9tZXYsr5w4RGYXDLLxvVVVrCrtu25YviOp+kVfWZR/WGaa9kuMF7G+KbCuolYeK6qQF7X/OhKumVgo41UW1uytlQdUDt6o/Ruyt0T72seK9sb3OjRTcjPlPTNHWR/9rMoWfWV3MHfwIDlTr0tmygD57aoi2dtwaJk+Wrg+QQ6QCfoQdJW3TR8u31+6Tcrq257/tToGT8rJtHE42eOFL8p/5HhksZtA2Z+tgkxHdOnYApmgnVOBChzbmEEZ8j868NFpTVRhOlyGZ6XLK9Toecu04TJyYJp99ko1at44ZahkDIgZoofDRaoUv2byUJmvwt9dpPXvr9dnoLxVz6Wrj2OsXu9L1Mg+d1R+/BWxwf6UoTmSqscRhWPh3rCN089xnE7ncr7eh9dOGipnq4EXZIINheBNU4fJJSovwbFyOEzJzZJX68B0kd7jlP79ZKQaQW+aOtyUXJS5w2HMwAx5pcrWW3XwK1TlpDs5IX+gKcYMzFmqBHQVGOrvnzXK5IPuY5IqQK/R+3KyysVA/V2Ug1fr38gODiruETJEn8bj4d8h5+VgTLhK+/K3aX+IUzDIBErttdpX4eDjfmDQHy7DtR3/7+zRpkxy/2YWxPpbjKHDcUYhgtPys+S1OlZE+87uQMXaHH7X6pg5S8+jKzljeK5cMW6ITNdz768XASPlqvGDZY6O51w3nDlvnz5C78cgu0eMw29QmbhG79OYQenxb3GOFcYAdAT0pEu1D8LwCHJBG0Be3qC6D3KRMeDwx2nGBdr06cMZ//vJ5eMG298YXocL8koQIDjruwv67Ffp2Md1OJK+4UhhrECPRQZHDky3/uQ0vX6MBYl9CWMw1/JN+h56FP2N0/NxTbeboGN7qyqsn5s/Th4t2iufeHKDvPXBVbZ9/In1ktKvv7znhJEWQYgKG95glEW2GTpYTdDvIQp3uOABmqwdSPgONgY9BDyRnLSUg/YboUp2eqTT5fko/Vx0n/GqyGSnxY6XToHjRbmZmpdl70/RR84HJR1PdvSzbAymRFnpbIbr76XpbwxVpYYBl8+ggBZmprY5Bz5Dx84AwnFzTnSU/C4ertH6vSgU7UGnxncSeUIB4noy4OAtxyvJ93Ks0/V73j1zpLzrhBHxTzqdCdcZ5fczJ401R8q/NpXslwk2vNYTsjPla6dMsDYV2iLtifYRbQ/tteeOoM1wvxPbFBvRtURFY7R+N22K93lELjj2AG2WtkcbZB/klPMKe9A+aef8Hp8P7+fq6/wWf0ePgXbH/nwvMoEjCNni2GjXnD/XAjmZEfkcxxCNzOKB5zfDteK3OzKW+X6+l88TqSHzgmgWx040s2WfmGFBBJho/NseXGn36LYNxfa9nzpxrCnSzrFDu7h4TL58ZO4Y7b9T5SvPbt4vEx9+fJ3cuXmPfEiNPhwI9HcB7h3jTGgPbLSBaB/+cjBeRD/PRtSEthcFRZQ2HPahndGHRynQY0+UU9osDjfaCg4IPsfGbyA/fGeatkX2C2NI2PibfhuZoa3TPtmGxuUiQF8Q/RzHEMZUrgV9SXSM4prhhImI9EEw1pD9Q+SJK8nnkAfGD/qC9+gYQTbDX9cV2z363NMbZXNlvXxs3lhztDjHDn02coFRTkZBo96Pjz+5fr9cfOXZTVJU02DOrPfPGrm/n6Ofph1G2wPtjfZ1uNBWTR+I9NP0u7TZ0K4MbT+0o/HZsfZHv85vRccKniOnYTxhM/nS4wkOUY6X19D1+A5+N7R7HHnR42Dju5AZ5G+o7odcsS9OU/pvIOuMYwljDd8RlRmuF9/N78bGqIH2d0cG5yD9XvYpqWuUmuYW++55Q7JlZ22jtLQenLzIsbx9xnCTESKeTu/A01DjJDsN9dOqTBGleGZ3pRmK5TrQBJlCmE4dlivfO2OSdXi/WLFD7ttaZu/9+/LZlioEDTpCrSirkY8+sc5So04fnitfXDDOOpV3PbzaOhk8N2O1E7jwjqU6mDWroTNSPjh7lAlzoLa5Vf6xoUTe++ga+/v358+wNJsU7azw3AV+u2qnpdIwSAKe6I+rAvO6KUPtb3haz+cnL263NLUROkjfeM407UgGmLLLQLqzplE+sGitDeivmzzU0kyj3L1ljzyyY691dB+fN0ZyUlOkWZso53fDS9st1Ylr92r97CQd4AOfenKjGRcNLa36nflyw9lTzdjL1U4vXZWRR3fulfc8ssZSGRPhepBiSNrt/y3ZLHO1k/v2aRPlHxtL5GfLt1sn+is9DzyUpPIiMSgZ73p4jSzeXSEVL5Ou1ZnQcR+vaagMwDeeO83SRH+0dJv8UY2QpshAw8BKm/m/UyfKs2qwfEGVsNXltZZ6jQFJewzj8M1ri+Vdj6y250TDuMebqurl/Y+ulRN1kCJtkRQh0rNL65t0n0L53zmjTTmN8jNt739eu0sKM9LkU2rEIlukWtI+c7Vdl9U3azsplh8v2y6b9fuBAfaWi2eZ44SIPnL6nB7v5Xcts/N51cRCec3kQotEZGv75ntItf69yhdt8PFXnXSQQlHV2CKP76qQzz61Qd4/c5RF+1K1/fFdi4r2yveXbrUUOKKuH54zRpWM2Oe4Njev2633b7ulwpF2Q3SjWmWAa8X3fnrxBrvOiaCsXKTX9YdnTpLL7nrRjp/v/4h+/6f0M016Th+dN1oqte0jV43x+0Sa5Lu1j8EpdLL2py1JGl6O5zRUohrv0n48V/vzc/71vPVxQSzQJ+mLfnPudJmlfTX9+Ne1D6P5cK+IOKIQAp+hb/6LtmfSxMicWHztfGs/tAEiYvSt9+pY8714WivjxY/PmmyfD2yparC+FvkjBRlDlWkLG6vq5Ewdg2CTGke/XrlTf+9A3/E+bbfX6ZgXjfyRovaH1btMyTxR+9efa78NKOSDM1LkxT011r74nEX19XcCxXVNcuPKIvmPGst/vugEM/LStfHvUZm8b1uZvOWBlbbfr7RPebNeB6A5riivkS8/s8nOEzn4Wfw3J6q8orivLKuV/31snaVftzd142cLp5pDFLlaqd/1+/Onm9H4Tx0vNuh5//nCE/Ta1cu3n99q6bwYq/w+xj7y9qd25K0rOJ7TUDEUMbwZL57Ytdd0E/rCIBcYWuhJtDlS50+77TlLd8S5TfYVulVgo94znFxffXaz/f2js6ZYRPCB7WV2D9GH0F84N/QBxt5bLp6pv5+9fxzmd9Eh0cXQ1ciOufequfJIUbm1S5x89Nc7VKau0P40jBXoFsg3skp7B/b5+/pia197dGziHD6iYxPtnfaKo4Lx5q4tpTruTbTINsZfgFTTs25/Xq6ZUGgyg7z103/0G2/Se0TK52kqp7/Ra8fvM9YwlYYU6jfeH5MZHIFcJwzAHG3rXG+uBzL9H9XREjlFr9f3z5gsa1QeGFOHqJFK9g4633VqvGM0Bhg7n9J+B8dOqZ4T4/BHHl8Xfzc5eBrqkTPgK0r8eZ+GOQUMojx2tX6DZ5P00Py0VBM+Bu7m0MspzJVjwKMTvF8HmzXldWb00EHhWcaT/w1VCBAy0ltIP8WriXeTkD773Lm51Lw2DMAIOsrAG3TAQnHerQPzB9Vg+7N2fk/uqjRP6RxVNBDeF0qrLMWGdMt67VzoIL/53BbtfPvba+xLJBRP1OdOGidjstOt8/k/3WdJcbUZuWP1NQzQXdpB4Nk7QTurp/V3frB0ux07nSZKCIrOTfr3d57fYscyNjvmTcb4+pP+zTXBgLhFFd6vL9lixt97teMk1Yi5BCgr92jHjFF6xvAcVfqbZZ0aspNysywfvkaPge/l++/XjpA5JuEqL9COns6COTsXjy2IefYKsuSV+t18/wx9jjKfrx34cyXVpjj/VBUtvPjA/jxHoaITThYoM0Q3iTIlg4d1sCOyhMHe1eAF/aAqnuuZK6Jte+3euvg7MRARBp37t5XbwIbSSgrSu/R64MHFeKedY7jQ7jHuue/TdcBjMGMOFs4IBjDkD+NxQ0W9pdJ9+eTxNifl58uL5Hq9z48UVcipOqBiPGJ00ZaIoqF4M5j/WpXUv60r1qPqJ1eqvFSrUomjaZa2mS+dPMGUki+qQkqboY2coQP6mSPy5MXSaosM8hzD8wk1Ar+t7ZMoBNGaT5w4Rgf7Fvnys5vMMbNKr8U43Q8Dd2mJGpK6P04fBvE/rN5pyjKt71qVs8vHDpY7VO5RwDkXBuzTVR45b64b6VWk0GEcv+Oh1WpIFtt+UQfK61Vx+YQaDBgApPsiWygWDPykY9F3YVRwvijk/9pYKiWR+SikxqJEoazdpEZA17eaGNxPZDoZ0H/SLzbrySXj/Ejznav3H4cAc9ETf5N+EicDfSFOBzIj3qn9Kwrf89p3oVhiyCAjpJziLNiqii1RBvZjfh+GEYojiud6Vap57To1UHEuLNU2+2E1nuhLa3WQmZibYc4EjGYyRxgXMIgwEHHGsN9k7R9RuDFkme/6IVV2STtbr/L2UVUMaXuMU8glbYU5k6MHZZgjJUUNPubrf/mZzXZOGFq03ad0DMHY4rOMLcgmsoRCjbEwW69RucrG31XxR6lnHPmdKvr01f9U4/ZLT29SJbzE0hMn5mbaGFXR2GK/SR9w75Yy+eEyNV61D8Ehyvtca/rcS8YMlhvPmy5v12uCQTw1L9McnZwTRjdywbUgivIjNZDv1rGBfoOxgevKdWYcvmfrnjZz4LsKxk6ON1nQt67Uc0uGg4g+/FWThphc0N7QB9BXAhzBHtWJntldpW2/VNZW1MpI1ZPerDKBwxGn9pfVOGQc4b5wT8mmwKAkpZp7trEyNqf9lWp0sc/yPTXWpr6megPROIyi76gxeZ+OR0TTTtK+D8c0uhhtGvlhjvv12o9/S/UjZIy584xNW1SXIeLHsbxjxkjT5X7y4g47j0yVmUu0L6/QtryLMUX1J/oAvhNj9DeqM9KuaLfnjc5T2Vwvv1yBU3O3jhd1crW2b/ptjn2rjpFETZF5dD4M6gtHF5jegxMfZ/CNK4q0P29Vec1VWc7RPqTCxhfGiqFZqfLwjr1mSHMdV6tcRGs0fFZ1QMbO102OOaUY085XI/1yHROJHDIOkiVE38C4yfd+fsE4G5MG6PXmnDCccdwkE/oTjtM5fJKjcfYCuBB4iYimdTUYUyiG5Y1NVrAjGj0B/qxqarZBmsghgz9eHpSh+7btMePpMe0IntCNfZgbMT3v5edp4AX9kxrEv9TOgc+zMSjzOoYlERGEF4iGMKjRARHpu3HlDpswvkB/61WqQNCBYtjh+cWwe1T3IZq4em+NDZzk+wc4v2W6Hx6pxaqEoIz8Vc+BTuq29cX7j4WOiF+nE92hygyddZ12wnQmGLF0eBepgPOIosJnMKZ/uHSrKUykap0X6QAwHBkUUKA4l+ggVqyGI0VzUKJq9fsYCFAAH9COP1PbwVr9LMowgw0DBMoIHRrn212T1bk3R5JC1hkQ5c6IeC27EgYPvI4YNh1NxOfao1giF3j9Z+tAyoDEIPjYTrYK+Z0aKeyDgc+AGjy27YGCQTEpDLPf6iCMt5bvoN1s1vaXp0oCKTYBZPNWbbM4SP6j7YEIC99BVH+BKo0YSxzTraqwPri9zL4LJRT5oO2iiKBYcy8x0jA6UTZIdyZ6R+T8e9qekRU+i4xzrqQD4jBi8F+nCgHRTTzitGvOn4GZARhjmTZMu+aa4ADCcz1YPwsowBgdKBLI4m691lHW6rV4QNs98tBPj/FpVRxwPG3U15ETFGCMQD6P8o5yFYXoDHM6t6oydHCv1nUgEyh6yYKoF+lYIYLb1WDUIRsY+R1Bf0V6MPcWxRbDnuuP4nevGigoxTeocouih6KEkfZyLNOx5U9rdsvPVTZoi2wYkWSH4ETjd8J4QZolBlLYb4nKH/cEJZxdmH/MdeMYY/vstUwR2h/Hw1ynAOeA/JkRq/3+It0Xp+rvV++0z9KuiZizXyzCj0OvyvoM+mqOhSgMjiQMVr4LuQi/yzGQWcO4GUCentUxhT4+VjSqWWU91oLr6CO0L/iXyhSvkwWEk/XvKqsYAKRAYgT+Wa8VYw1jE8Y7Mkmk9nPzx9tzvpu+Jhlw7ZM9VmDkENlNBuhEpB5jNOGk474kQntAt6Gfo92jP+Ho4n5i6BEpu0f7Zu4Z7Zg56S+XsYMzgn79B9pH08fSppAtdAeclPTttElA31ivOhPZY+yHk5NsJRx/8wbjVMuxcQMHAxkBtCEc0DhIAIOLuY+APoaxxbEiF7Qxjv9bz22134+17Qpru4wDOEgIfCAj/I3OxJjIeXId+F6cnQ+rkcrnkF10SeYdnjsy3yL4RBwZbxmj0JUYhxIL1eBsfGRHhelqHCM63R91fED34vojd/foWIaDjegsUV6c8Di9kiULieBoT+Z4cbzgxmIcBjI8ndH0r66CwY20ATowBsswKHUEKQzn6oBar/s/XlSxX0Hj8wgiHQCeMNLZDgXKHZ0aAzleI6KSl2nnQC76vgTVDuFGYVypHQSd3mL9LAMgCutlYwebB5pPLNfBkk6EyNPO2gbtrKptf74zwODKe3jcwn4Yi0RB+A48x2x4ig91/RFwvKUYm6HyHwo3nSvXZHJulkVSAgwUpPG2B8os14IoEl52IjykHP5OjcJivSecB0Y5Hbj9hl7nQylryYCB2OZtJrGfIxqRrJTX/tLPjFMMmsOJ1mJA4XTh/mC8YMgBFWxprygUl6rSnB8fvNsDg3OVtpOfLt9uEXPaF22RghWcd7Q54pAgXQ5PK55SBk5khAgl8xgxEjGWaP8MhqG9MJCilKC8oaCiULAPgylGZFB0iGbglKDa66yCQWbkkVKNd9omwHQAxiLtgsGXAR8vN9VacWzQ/hcUHqieyW/R9jsCZQMHERXt+CzX9TeqLKCYoDAvVXknlRGFPVEeSEHFCEFhR24S+5Sugn7vSAp4HSs4FMmwSJbCQZ/Iv2jk5FDg3MHxRyYGbQ9lkYq5OP1w+KEoMg/25UAeSJmk38dZyXhxkiq6OQn9AXJBX4tzLYBDAyWeyB9XiUf6/aAgIqscDymhyDBtJ8A+yBewH2l9pILTpkk5Jx2aNt3RfFugb2Bs47dRkl9SWQD73aJyG0u5DuxHKyWTp0iPj7abCP0LzpzrtY+gb9mo8k566a9VMX5Iz4Eo512qbGPMIi8B+h+Okfu3Qn+fPm1EVrpFeroa+tEwPy1Z0F8yXiQDdJ5BqSnmiKYvPRwwoMhe2aQygYGE/oROgsMNhzQRvJczsGnT9O0YQZkDBsiVOk5gZBIxY55gFOQCIwsjD2gnOMf5hUm5GcIUH9oD7Ym+mvGA78cBgfzgjGcuPeA0R5ZJ/wf6cAxLDDmMU5xDjFvIEd1ER1eE6DdjXHVTs9yqBipGJCCHyCNOsFOGZVtWCroa+iltuqO+h8wdUmZxtHDsjF8/1vEBfQq9CqMcucBoJevmVDWQyTRg/OgupzvnmKx2ejzhxmIcBn7m9kVzv7uK0MHR4TGgdKRy0HEx0OAJwUjDI0bqU4DBhxRJUmnwBKE0HgoGRibyE9342NzR8vVTJsgXF4y31NXEtEYUYYQ/SqN2VAy2wZvLeYS5SgE6BeZCRaNRdMqJ0VOMSdIWSJn46cKp8gU9DgxQDPaO4EoRdazUDo4BPAoeP64p1zNA2ml7gz+gPHMt6JBJM8IQQ+lkribnx3VnXuXLXdNkwjFjlHTcYjqfgowUm0yfDGgh3ENkEedNe6CoowgBbYj2QlvAsIkaJ7Q52gTH/3LZAsgZ7YD2+N6Zo2zuBemwpK9G27ENnjrARdsyx4sBxneg9NJ28LAS6eEYgDaIoYWnl0glfQyKROJ8KOZBhaIDzI9kHibFMojSJ+ghB8Fv0ya4Dno4+6nS30VBR2boa4A+A6XkYGk8AEYQmQHjdUPGAScOjhjeS9cD4fiihVK4H1y/L6kM81kiKERZosfSlWB8M/czWXDeOAWSVQk51ooYL+xJu3APgvHKvaav4P4nKnkoaLTZcO8OBXLP3FMif99TmeD+MgcSuYhCm0IRpu8P0OYxyEIBiyw9HuShUpXUKBixHCNtNIDhFiIYnBFyhVzgwGG8IlLHRgpiRyDzyCLjE7IWoI+w7BD9/lDEA5DF6H5RuK6MnSHNDvnlb+aIknLNWIFDl0hX1BBEDj771EZ558Or5atLNpth8bbpw2XOkK6t1grILqmayYT2wjVPBtyp5tZWGyf41xG8bzqW7sIjf+OMjEIaM44VZOblpAK5IUKG4XndjOH72+Mn5o1p45QkYri09EA0jjaGU5s2OSQjzdp1qgo1choNGCAnW6vqzPgMMs2Yw/x4ZBeQd9ocbZBUVo6DaTWkntM2Q3+fCEYg7zNWEPUMYxnHuMWWAWuxe8j3Y5iicx6KEQPTrD2Tls2epKgjJ+hNXG+OH+MUI5WIKvLxqxU7TIfrLnAYcA2cIyM5o10vgE6AaBtFYboaJkLvqWu2wWpm/qD9HUIiTIxGKessiFT86YIT5OaLTrC5hZ9cvEHO+/dSy3cnxSyZ/PycqVao5WPayTKfZOHtz1sEg047GeDhuvmimfLMqxdYWgaFQTieOy+fYwMBy49QqOCbp/WcSdC0hVP0WJMYRJHJ2gZnHEaKc2fAgImSScpkR1XXMFjw4nZkTB4NKHlPvmq+3X/awg+WbZMr737RvMJdtYZhe1AU4a4r5sizehxEKb/7wlZ5zX0rbA5VUBK6mreoMXDLxSdYgSfmMVPB8b6r5sln5481pxLRpWdePV8+Mne0Ra+AapDI0Tg1oIjQ3qBbMpk/hDlzyVOMaZtci1D1uatBgUOnO1SkiD4rFoHuPD4we7Tcf9VcuemCGdb+3nD/CpuTSJQ5WSDnn9Qx4m6Vix+fNcX6PgrRsR3peqtHC8WtmOv/1LULTDHn3jPv6t4r59rcLyJSFEy54/LZ8qE5o+KfOgCK+dLSKouWYiCw5FJXE1LTkwn9MjpLMiBtmAwgMgow1jsCWZ0z+MiWzjgU9Hk/WThFHnvVSVaUhqgbbfGcf70g25Oku8AMbXOfPmmsjVtfOXm8OSff8fBqm4tO1lVHjo/O5ssLJliRIdalnqlG4n+umGvjKM5WUsAZF+5QnYoCgYytS/dUyfKyGrt/3QXyylJ1zpHhxmIc8u2Z45SMFBHm/zDhmlD4KyYO7tDLSwXHH5w12daIw/OD9zUlsitRDyqekarAPK9QYasjrFqkdqxUn3vNfS9ZChzRw8ToIJBSEiILAaJv9Mt8hvQNopGJx05nalHQBO9dgO+kaiHKHRO1X//fFfLt57aY1wwv16G6OMoNMJcTr1CiMYFXE6ObFLzDgTmQX3pm0/45jV99dpO865E1lh7L5HOq4V1y5zIrPtJTIIJy7uh8m7+ULChOlCylgwgDKc1EtfCatgee1K+cMsEM+dg8rkZT2vFeRj3MWan9LWJcUntwJDCRE3SAe7MaQBSFZrK/FX5Zu9s8qokGGoqrzR+LXH48uDh9OHbm2ZK+jVySYhRSXYh+4m0lqrinITZ/JBGUO+STdT0//9QmebseB6ltRGcOdfxQb8bEPrsOURsaOSRyQkTzcI1N5tRQNIeUQhbXv+TOpXLpXcvkP1vKLFJywR1LrToqc8hIv2Vpn1+cM1W26vNPPbnB0m1f7ng7m7NUeef+JwuuK0WNEvvHroKpAKSDEbVtD275ZXofqPr5MTVeuNdkd+Rpu0ssTU8kALk5VMSANkREcYZupIR+9PH1VlkUhZR2ntiWiCBEMzqAPpr2HzJAaPNEbpCBKOzHZ2mj7cGx0AcxvlFo7f2L1lqExiKBh2jTtEHGKcasqDFBH4HDjajbnrqm/Wmxh4JCKcztv/o/L5qjl6ImF92xTP738XV2XH9cs9vGjrc/uNrS3/992WyrtojMA0eJSHDFOZKojHYVZM1QVCuZMF+bInHJAIPo5y8VaVsUK9rEPLhEcGK984QRcv3Cyeb8xdFI241GsQEnLAWPcAy+XNeVre13dsEgm4f4/kVrbK47bRF9J/GzjBdUxA5z3tGp6OPRmYjEM16QrcX4EXV+DlR5oEBUg35hR0Yfxg5TFegb3nD/SiswSCoo8nmorn5XTaOOmc12HhxLyChD70XmkVHm+3akvyVCQSj0OKYj8fzae5fbPGdSv3mk8ivVUPeqDOGgf4salQ+/Yp48+soTrfYFFe1J4yWAkQzdG6gtMC0hO8J5edq3UvogDKCkoZLaglLelWDQoBTvqm0wZeu66SMOSndkAP36qROsEyRljAgHiiMDH0pKSL9hML5i/GCpbWnRzrO+TXpFIuT4M3CSmoaBhMFHZTiKECQafRwPkQWUTToyJj4zH4lSx39bXxxbykN7JSaMs/G9pKsxaRvFm7kx7cF+pPKwJAZ59xhr27Tj5zswSlDwOwIFgPmKDMJ0woDyzkDAHBwKMjCJ/HAgRZGOlk6RuTDMUaTDI7JFWgoFE8irT1blupcDRZFrhOc2OrB0NUTaSTujollX/y4OEeYFodu9ZvJQS1GOgjPH1pLTgZ2J+Qy2DLoM/pRJD0Ycnn/mNjFXlraKQdkRpC9ScIr2H4rMoIQiE3ggo2npqXr+KJl8N9FIogSkiE7NzbJ5MM+XVln75DCofsj7MC0v0wrtkIJHcQLmciSC55tj4edof7RFlF2OjYjjoVKPcRIxfxDPPs6jkM5Ke8HoZu7Z4UZIkQUME+SD548VVZhCzb3BIEQm2Jg/w3FRXZBCOCgr9FOJqetdCcoXKZL0UxhGyYKKh/Q/KMcU8+lq6JsoJMG9/MGZkw8aK0gre4cqxKRoonQyttCnmdNF+2MMPvpUxhSUM/rJtdqnEek6FFxbFEeUa+ZcUVCD8YUKw4mLk9MvcC2IugVQRJEVMkXQXWk7pO/RRgEZpwANFSQpSEOxkPag1Q9S+caNSJorc3Jx0CBfh1okHXlA9rGJmccVCq7RR/A3j5v12Kjq+3IQBaH909+Qcs78TGSA4juMgdQA4G8UZPYZqeeJg5c+AkjrQyEep68TXVyfUOW5syElnLE6mQ4UoC1yfxPTlLsC2jh9LfO8ubdkTUUNRq4BfS5tjL5zW1WDPLmzwqYr8B5jC20cA4X+g2gvfT/G1qFI0XaDo5o2wTxY+kGuM8t8IWNR0HWoYko9CUBu0fUYX6wglbYf2jTjB+MNDm/2pegNayIyPxbDrT2QTZxVDa0UZqq070PeKRRFQTfkqz34PY6Zc+dYwvxjdC+qX2NIPqpjK+Pq4cDv4kBhvHhUx4pFuqFfoieuKKsWCkohF79fvUu+9fxWm+/+r02ltlGgjSlUyMR/VW8jg6Ir4X4wJqI/k4rqHBm+dEYcZAuDkTmBeAsZ3A7luTxW8BjhOULgKcmPd5VOhw6PFFHmTqEkMLGfalcIL+WQ58ULZDAAMXAz8GEgUQGRQRtFmUjGne0snYE3idxxlFjmdPBbdC6xtD4G2GarBkkBGzxtKaoUscYOn3nTtGGq0A6w8ti3biiWEt13pH4PSikGDL/JXEgUbJRuIhGU+KdqKoo4Xi9eZ6Cns52pHTTHy/OTCnPMGOW8OFbm1aDgooQyj5HjMI+ZXreaplbr7FGu2RfhZy1JDE4iQgzcGFacF8UHKF7SkaLMMbN+F4oVHRoRIQwSjFKuOVGiRPgMHSvXngnmyVg6g7ZB6XYUELylHYwDXUIwEGmvTLzn2nSVVBDhZmI/k/5pm5PVyGLeBs/ZKM/N3DkiXD9ets3mPdDGmVtK2iYDJ+2PZSRw/OBgofovC/mjPFHoI3HpDArUAPeTuVJ8jnT0hSpHtE3uN0oESuWpqpQwZ5TfpO3xW1TfZRCiWACT+nE24MFGFrhmDODnjS6wuU20F6qdolSwYDFygVziHWf+G+tL4gjBAYLiT2ohRjrtGSWC4jNkJKDgnDMy3xR5IkQosrRw5JCIAn0D1SeZI8J7N63eqYN6nZ0T7+MAeWJXZYf38czheabsY/hxrVHGkDMUF0q7AxFZ1v+6SM8NeX1BlXj6gnCvcAjh6KJSpF6GTsecU3ourDnJvU3MNOhKEAmyGLi/KEpF2jcnRts6E9oUcoeCg3OPvpD7yHXGofg/qvTRPmnbD26PFR7ieGg/tCX6e/ZlTUQifChqFEqjjbW3dAbOAYxDCsDQJokG0qb4DtrjOJUnDGbu+cDU/vY6YxdOCto0f9PmWW6G36JADjJDO2YOMfuwli39Nu0XQxHnI/0/SjzLL9DWUUCJBJJNgKLN+MV4hLJH/04bQ2lHjjHWWCuXyCmjOW2SIiZksAQZ4rNz9LMX6/jG2EoUHAOYawp8R2J13wDVXyk2xXjKfjhoqEDOMks3rdlthgvjADoEYxzRC1oEhjr9CcYM/RUVw3HUHm4GzJGCc4vlnzgnrk8y4bw5Z5R+5oXSBrtOKmI6G7oBehF9Ae2V6037u1Tv8RkjcFyL/HX9bqHg1vbqWNYHskN/T5uk72DpF+SLpdNoq9xnZAGjijTT6NIZa7RtEoFDLjhf2iXGHUtdYCzTZjl35OxtqlugGdAu0JXO0j6ZQjQUR7pT9SzaLf0IYzptH2MRZzvthb6FQmPIJX0xBZ1om4wxtFlkDblmCQ9+i4wuloKgjyeIwHVHhyFiT7+MAU8/QjEd9uezHA86KPMNkQnG1n9vKpF/bCy168l1pK+hgE1HcF3OHplr14NiWCEAga7IGtTIMjBnHx2Lth827gHONp6TwdVehltnwb0arNeXZccYG+nPnCPDjcUE6HBI4SEylljtrzPBkGJApAQ3axGijOElZ2MwpNMi7YZUMKCDQBlgfbbzVFFkPwZBBnrWlKJToaNEQUD5pKNAqeF8MHoZtNkXRZPISPgtFEC8TXRy/AZVP+epEFMtC+8dxihrAzFv43eqdP5ixQ5T/ugUWKOOTg4DC6UF5fmp4grtdIutk6UzRknme/GM4wWkQ+DcMAh4D8P0NFVGiWBhYBJ90f7JBnMUFtL56EDp4PEEU6E09r051imxxhVVyUiBo0pluj5nbSC8zo/EB386x/agwyV9i2UKuH4cAwo2gwIdG8ZFInjkMJ6HqxJCqh7R4a7s5DDW6OypAMgAiJKcbBgQiGpwjSgM0dXGMcoYCiWDMOcc2mpu+gCbp8T6hbRxQEkjEocsoECerbKBZxUj52tLNltbZS1QlFm8yjgh8tK0fRQOsggLEUqcGChWKMoonCinKA/MFcQQxXFU3dRqBQxwPPDTKLvILLeDtE2cMywpUKQbHmfaNGlgKB5EGlirkjRX5Iz2g9wgX8glx4jySEoSCi3neurQXHukn6AUOXJAtcaYDKmxoG2CNCTuzTP63UTogbUWcTbhAMKRQSU6qgUz6OPkwCON/JGG3VGrZRDHoCWKg/cbpYK/WdoGwxmu0muN4o1iwkYfEe4TGwoVMhzSyzsT+jUKRHC/PnniWDOIglMjmeAIoOARSiX3tSshysv9oG/DQGetNK4z9wqF+bvPbzVDhBR6+ltkFcMFpwT7YrzQ333vhW1WwRA5xqnBWoFheRbaEoV7cCawzmldS4spw9F7+6C+Tv/I60TscQigePGbe1U2Xj85tiYi7+G8u0UVTe4+MsYtQgFlsXC+k9/7wdJtVl0yVeU9X/8+UeWSY6eNotwCBjmOI9oh381YQxog4zPtmrlr9P0o22TCoPSyhhxjGdFunIthzEMmyUChxD9jFAYz15DfQhGPFumJwjmeo7JMz8d+6AiMtfTNVIAMkRiUcyI99DesW8fvYowgI8xB5vp1hV5B62ds4JjIviA1u6NaCF0JU2JwLnAPOU+Mnq6CMQCZ2Kn9nOkhcZlg4zoQEWdayW/0/tBO0ANoSzicGSvov4mIo8tgEP1F2yHFyYh2Y1QSNUNvYT8MX86JfpZIGEYZuge/xW8zjtDP0Y/TT+JoZvx6StsKugh6Dn0ieuVnntpo4xayg1yzxiHORaKLyAdOFsYU9D36fyKPjCFkQoXoJ+dBBgjZHbRpnIfIM2sP4yDHaYBMcCwYSlSJR2fi8zj8yG5h/VwCFSyVgROH/Vm7mnEJ5woGJRFCxtKOwHlKmjhjF8vZcLvP0e+kfyGIwbXtiPmq/zFdhDENWewq6Hfoa5Dzj88dYzpxd4wXvZ1++8gZdA4CQaThf2bxhvgrXQdtFmPnoLard4Q4Gh1X4rIaRNRQAgx9i04MY4X9EAAiMXwXwgsMIPzN3Cde4X0G5gCf4yfYh0cMAfLY+ZtfiQoVXtyo4sdhMMcqOihRAYx9OC5eRennK3gtOnDwG0Q3w9fz2xi14cjCOfF59uPow3dwDlyD6LFxfuE3eY9ITZMq1ex/4FcPho8zVzV8L9/H9UIhoNKavtQGPsPxsF+4Px19/7HC8RAxIcUFzzQDYHfBAIIC9+5HVtsA3dVwfWlbXO9AaCOJk+MZDNMGHGgPifvRPrln3Cfz/sf3R0S4z7xOpITP21foC7xKm+A4yDbgPkfbqz6z5/xWYjvTj1i7tdTR+D58nkEewvFAeA1C+0uUJ86F4+OR34FwvkHe+BdrlypB8Y/zm1G5C3LD37zeERwD1wh5Yr/Ev8M+HGe45onw27HqyZ0vH0TacWL97+zRcqEq4+0fQdfDeaE4oth9Jwlzm/VyW7+2v51GoF3TFqLXur17FN2P17mWTfqa9b36N59BLniNtkKb2j/eKLzOj/M6n0F94DfYg6YR9k1se9DR8bAPr/C6ySWf08+Hj9rr+rnocYTP8FXsx/eE7+cfbZXXwMaQyGeDzLAPL/M+0F55rT34HWSLPfks58fxcFzIMMcQhe9EZvicwTFqP5F4jzoLzg+HDnMlX6HGOI6M7oJUXaq0v/2hVR0a350J15l7H20fwD1K7OvYg2sV5upB4n68R5oy9wo9xPpy/Zt92GiPTAki8wmQAcYIPgPso/+ZrhbamO2rrzGu0F70ZYOXQ78d/zo7HmufurFbaNeh3cY/aufNsYXP8Z18FvjNoJ9EzzfIG5+1MYQrov/xsajM8BmOi28Lr7UHv4OzikOIyhswLupPdcj+66z7JI7pnQnHg2H6KdWjGC8S56w6h4cbi+2A5x4lAM8OCxInI9XQcaLQWeO9Z4I+aTCkGKLYdRd0+qQXPbSj3Aok4bFvL/LqOF0JHnNSiN84ZahFehILpiQbxgU840T1WFOMKHdUOXWcrgZjgiWfSOt9xfhCYW5oUNi7A4wOIq/36xjx21VFNlewowJGjtOVECFmjXLm1rPuKuNF1FnlHD6ehtoOeF2I6JDSQvojYz8Tcb3Dc7oaBnnSFJl7yhwjm5uT3b2GItC/4gmkchxpnaRvMQeHVCNXjZ2uBMcJgzyLOjNv99qJQ22uUU+Yd8KxkQbLWEHqFt58xgoqJDpOV8JYQW0BUhFJ/6VYHn1zdxqKYJE3Ha+YG07KHynBpPl25bQexwnQ/pguRHosc6CZ5kXhN6ZghWiwc+R4ZPEwYM7Pg9vLbK4GRR/IR6cYRleGzp2+AY4JDEEUTuZ7YCgyt4z5C8x5wCjrid0bc0GZp8d8J+brIBPMbwjzjBznaKG9Ey2hoAmpdcyXZb4PpdcpKEQRos5aN62zoViLzRUqrzGZYE440UZSvxznWMAxwVjB/CvW1GReJPOwmLtLml13R9k7goIvZGkxfy8mE7Gxwp3vTmfBeIGByHiBUYjjDociczKZV+ypp8eOG4uHCYUtKGTAxPpnSiqtiIUrxh1TVVklO4p22PMxY8bKwIHdN9+uJ4MXuDAj1To3KkzSwTE3kcnqPR0UYJRhKtBSEIjqiMiI0zFbtmyRuro6yc3NlREjRsRfdaLgGWatMdbbZKkHik5RqIUsj94AhSsojoFMUAWaub4eaeyY8vJy2b17tzDXcPz48ZKe0bVLV/VWcJBQdIQpCTPzB9pYQfopxmNPh3RtDEXTn1QmKDLE0llO+7To9dq8ebM0NTXK4ILBUji0MP6OkwjORQq3kX5NMSAKCi1Q2WDeLkak0zm4sXiYcJGYLMxkXCYa89wvXMfcfvvt8ta3vtWe33HHHXLeeefZc6ct9GcoSkz2ZpI+ynJv6eLoPZijEjYPoByaCy64QJ555hl585vfLL/4xS/irzqJ0P6RA2SDRyLwvWXc3z9WqDAwXvhYcWhuvPFG+fjHPy6pqamyaNEimTVrVvwdJwrNX8Vgv1zEZKL3jBWMDQfGCR8rDkVFRYUsXLhQNm3aJB/96Efla1/7Wvwdpz1isnFALtClkA2n83Bj0ekSbr31Vnnta19rz++//3658MIL7bnj9GXOOOMMWbx4sbz97W+X3/3ud/FXHafvcsMNN8gHP/hBMxaXLFkic+bMib/jOH2TvXv3yoIFC2TDhg3y6U9/Wr797W/H33Gc7qFnTvxwHMdxHMdxHMdxuhU3Fh3HcRzH+X/2zgLOrur445Ns1t2zsY27EMPdvUhpS0vtX2+p679K3aAOVChSoPIHCoUWL+4JkBB3zyZZdw//+c57N7y83Q27SfbtS5gfvE/2yb333HPHZ84ch8PhcDi6wJ1Fh8PhcDgcDofD4XB0gTuLDofD4XA4HA6Hw+HoAncWHQ6Hw+FwOBwOh8PRBe4sOhwOh8PhcDgcDoejC9xZdDgcDofD4XA4HA5HF7iz6HA4HA6Hw+FwOByOLnBn0eFwOBwOh8PhcDgcXeDOosPhcDgcDofD4XA4usCdRYfD4XA4HA6Hw+FwdIE7iw6Hw+FwOBwOh8Ph6AJ3Fh0Oh8PhcDgcDofD0QXuLDocDofD4XA4HA6HowvcWXQ4HA6Hw+FwOBwORxe4s+hwOBwOh8PhcDgcji5wZ9HhcDgcDofD4XA4HF3gzqKjX5CamiolJSX2Sk5ODn/qcDgcDofD4XA4DhW4s+joF8ycOVOuueYae02YMCH8qcPhcDgcDofD4ThUMOh1Rfhvxz7Q1rlbtja0ygs762RxZYPUt3dITWun1LZ1SEvH7vCvHAFa21qlrq7O/s7OzpakxCT727E3MpMSpDg1SUZkpMiI9CQ5eXiu/p0siYMHhX8Rv9itkqO5o1NeU35YXNko62ubpbK1XSpb2qVJP1eWcUTh5ccektrKCikZM1amzDsm/KkjwBCl+7QhgyV1SIKMVD6Ympsu84oyZVpeevgX8Y9WJfwtqisWV8AXDcYPvOraOu07x97Yum61rH5lgQwaPFjmn362ZGTnhr9xGFQVGE8kJEhBSqKMzU6VWfnpcszQbMlITAj/KL7RqWZmQ3tniCf0tamhJcwXHaZD0CWON9DR3iYLHn1AmhsapHTyVBk3Y3b4G0ckEgJ9kTBYhqUny6TcNJlbmGkvx8GFO4v7ABODcn+lvF5fDWYUr6xulC2NreYgNukLQdfuks6xn0hRIZeVNETykodIrr4mqXGMoDt2aJbMiVOBB70TOHlpV50sUsW/prZJNtW3SHlzuzSqQYBR0LZ7txsA3aG5QS2nDhGCJ8lp4Q8dARIGDZKkBH2p4wA/YACMzkyRiTlpMrswQ04syZEc/TzeAK2jK17cWSevVtTL8qpGWVfXYnzS0N4hDaonmlVfdDhTdEV7q05es/6BV5QhMvjQcIBiiSTVE0lqGOMcFqYmWSBlXFaqzCzIkKOLs2R6nAZT2pTeN9Q1y4Jd9eok1sva2mbVFa3mKMIT6AsC8c4VUcAsR1fs7tSHnxJ6ObpgMPpC+QKdkZOUKEPTkqQ0M1kmqL44oiBTji/JlhL9zHHgcGexB2AQV6hAe2FHrTywuUqeKauV9Sr0PDLs6G9MUYfxzJG5csHoAhV4GZKtziQZl3hAi9L/quomeWxbtdy7sUJeLq+XpvbdFjl2OPoLBFSOU8V/+fgiOVb/HaFOZLIa0PEAHEB0xUI1iO9cv0ueVV2xWZ1EjGCHoz8xS/XDeaX5pitm5qdLSkKCGtDhLwcQaAMC6kurGuWRLVXyn82Vsqi8QfWHZxEd/Q+CK8cNzZa3jyuUk4fnyNisVAvMO/Yf7ix2AwzfssY2eXxbjfzo5Y2ysb7FjGSHI1agtAKn8etzS+VYFXqFqYmWdRkoICQ6Vcsvr26U21bvlH+uL5d1tWQDHI7YYUxWinxoyjAzAsZlp8qQAeQJgK7Y2dQmT6mD+KvFW8w4JlvicMQKeSmJcrzqiK/MGSUz8jIs+ziQDiO6gkDJMuWFG1aUyf2bKq3yxOGINVjS887xRfL+SSVqT6VZ2Wp8hN0PPbiz2A1Q/vdsqJCrFmywiDFGsk+SI5ZAoLFusSA1SX52zDg5a1SerVcZKJBp36F88eXn1lpWkbUmu110OGIMyo4oQ32XGgDfmjfayo4GElWt7XLH2nL5+aubbR2W6wpHrEG8hKzJ8PRk+fMpk2VuUaakDxm4Ul6qrwiwf+6ZNfLSzjqpaSOb6FzhiD0ImmQmDpHTR+bKr46fYPpioAOMhyoSrlKE/3YoKJ34z6ZKuWXVDllV0+RrTBwDBkivWRXvhroWW6fCuq2BKEeFB7aoIfzr17bIf7dWy66mdi87dQwIoLrWztdtzVO1OmpnjMwbsEgxaxDv3lAhf12z07IovnbdMVAgSNGo9LipoVXGZ6faOt+BADywUu2mX7+2VZ7aXiNVrR5UdAwcoDxoskbpcHN9i5w4LEfSBjCQcijDi3ijQPe6x9QgpnGHO4qOgQTUBw0uUZp8cHOVrYkaCJBphyf+vbFSyvTvDlf+jgEExifBi0eVJlkPVdfWEf4mtli4q04eUr5cVF7va9kdAwokMqWfz++olUeVJwh0DwTgSwKKD2yutICOBxUdAw30xa7mNqNL9AV/O/oOdxbDQKQRgXh8W7W8qEYAW2I4HPEAOso9v1ONABV2ZDNiqX5R9mvrmq0sm9IiN4od8QB4gnVQN68ss46jsQzsBbqCxmcLVFeQPXE4BhrQJZ2oMYppshTrPgvwIBl2KrPgSc+0O+IF0Oau5nb5x5pdsrqm2fSHo29wZzGMIPrwtApZ2js7HPEEmsmQydjW2CqxXGaM8bGiulGe3F5jpU4OR7yAfQsxTFfUNEl9DJvKmOHR1CaPb6+20iaHI56wqLLB9oOmCiSWgAeXVDbK83pthyPeQOb9v9uqbQu8mtb28KeO3sKdxTDI2Dy8ucqi1Z49ccQjKAdlH7fOGPpsBE5eUwOADo/uKjriCWS969VhpInG9sbW8Kf9j/q2DnlCHUX40SPUjngDfRfW1jbJq+WxXbZAp2xezTEM3DgcvQWSGjuGJWbr6zzI11e4sxgGUQeIqNZLihxxCsrdKPOJZcMAMporqpvcUXTEJaBL6LO8OXaRYjIoT26vNUfV4Yg3wBM7m9pjvm5xpfIh+sJ1hSNeAW1Co7EMLh4ucGcxjHY1wFfHuJzJ4egLaNOPsxjLzOL2xjbZ4qV2jjgGZdLlLbEruWvq2C2LKxqsGsXhiEewpIa1WbEEJdnoC4cjnrGurtmW8zj6BncWw2A91vamNi9BdcQtWD+4uaElpmsWa9razUl1OOIVW1Xxs34xVmjfvduMjTb91+GIR9CgL9YGMXrCGwM64h3sF13V4nTaV7izGAbmd2NHp28L4Ihb0Fgj1t1Q2dMu1l31HI6+gDVasey8aGsl29k/LvyBwxFngB9a1J6JJdATHmx3xDugUQJ+jr7BncUwUPxkbmLZgt3hiHcgWDHGHQ5HCJSBs17R95BzON4AgUx3Fh2OwxPuLEbAdb/D0RXOFg5HBJQhnCccjq5wvnA4Dk+4s+hwOBwOh8PhcDgcji5wZ9HhcDgcDofD4XA4HF3gzuJhhiGDB0lpZoqMzUo9oNfQtCRJGxJ78shMTJDh6ckyWu+BMSQlOIk6HA6Hw+FwOBwDgUGvx7IPfxxjS0OrzLtjoe1PdChjVEaKPHbREZKXnBj+ZP/w4OZKuWllmTyypTr8SWzwrglF8tGpw2RiTpo8t6NWvrtgo+0t6Ahhks7LwsvmSYY61bHAF55dK79cvCX8zuGIT1x/0iT5+LRh4Xf9i4W76mX+nQvD7xyO+MQJJdny1MVzwu/6H1c8ulxuX70z/M7hiF98Y26p/OCoseF3jt7AncUwDhdnkYwczkR+yoE5i/dtrJDrl22XBzZVhj+JDd4/eahcOWOETMlNk6e218jXnl8vr1U2hL91uLO4f3jPxGK5cEyB5CYNCX/Sd7AZ+5KqBvmj8gXyIlbI0jGfMjxH3j6uSFISBlsg588rysLfOoA7i30HFRwf0zmbmZ9xQFUkNW0dqi8q5dZVO8KfxA5vU57+1PThtuXVp55aLRvqWsLfONxZ3D/ML8qU80cXyDHFWeFP9g+Nqi+gye0x3u8yfUiCfOfI0WYrrKxuknvVlnu2rDb8rQO4s9h3uLMYxuHiLGarYfkBdbhSe1D+Z43Ml5PV8Gzq6FRnrFYWVdR3u5HumtpmebW8QdbXNYc/iQ3cWdw33FncP3xz3milq+FSnJoU/qTvYLuEJ8tq5MvPrTUlHCsUpCRaxv2LR4xSoz5BblxZJv/7/Lrwtw7gzmLfMSEnVa45drycOCzH9Mb+YqfqzOuXbpfvLtgQ/iR2gKd/e8JEadu92/T3kkqvQgngzuL+4exReSpLhlsg4kCAXQVNrlVbKpYguHj3OTPkSHV6X9xZJ9cu3SZ3ry8Pf+sA7iz2He4shnG4OItvhh8dPVb+d06pVLa0y9WLtshfVbhvboifaOxJaricU5pvUe/l1Y02vk31Hi0O4M7i/uHScYVy9sh8yU7uOm+ZiUPk2KFZkq5zyl6rZCfW1HZ1BtlHbIXS5K2rdsq2GEaL3Vl8c7iz2HeUpCXJ+yYPlam56d0GF8dnp8nYrBSTNWTVnyurlbr2roHF2rZOeXhzldyxblf4k9jBncWe4c7i/uGIggw5c2SezFNnKxrI31EZybZMJjlhsKyqaeoxmA3PfE3l9I6m2NqU7iy+OdxZ7DvcWQzDnUXHoQB3Fg8+Juic3n3OdGvstK62WW5YsV1+vXhr+NuBR27yELlgdIF8eOowM+rvWLtLfvbq5vC3DuDO4sHHp2eMUJorMcN4Y32LXPzAkphm1HuD900aKlfNHyPt6ixeeP8SM94dIbizePAxQh3Fd08sli8fMcqCeD9ftFm+8lx8Be5oEvj7kydZefmiigYrD394S1X4WwdwZ7Hv8FaTDofDEcdgTdj/qYN44f2vyRn3LpLfLYkfR9bhGEjAFwR5j7nr5ZiX+zkc8QiqYz7+5Co58e5X5JP67xPbYtuk0HF4wp1Fxz5RmJoovztxotx37kz53pFj5IOTS+Qnx4yTe86ZLg9eMMvKHYjQHFWcZdt2gGl56fLx6cPlT6dMkjvOmiYPnD9LHtbfPqj/ksH548mT5DMzR8h0/V00zhiZZ9e799wZ8sOjx1q2J0BJepJdj3O9Y3yRTM5Nk2/PGy23njZV/nPeTPuccd54ymT5rJ5/RHpy+EiH49AFtR8tnbulprXDXpQ3ORyOEF9Utbbrq0M6vUjK4RC4gPX11coT9eo4tu12vnAcOLwMNQwvQ+0elF3goLGuhXVc1N+Pz06VrMQhMkh9Q9zDZ8NbXFC7f86ofFsgTkcxOrMO5kcKiCxRnUm6OdapINva2CKPb6uRf22osEY2AfbV4GacOo6UYOYkD7Ea/A31LXY93nPuJH0l6PUwILine9ZXyD16/sOpQY6XoR587E8Z6pFFWRYgyUsZYmscK5SfWGfLC1qEBuEX1ozwPbRZot8dOzTb9g/l+UWuE2ts3208uaGuWR7ZWiWR+p11MrMK0uWEkhw992BZWF4nD21+o6zotBG5NhaOeaasxvYmDY0lydavgFYdT3lzu60Dfrm83oyJwwlehnrw0dcyVGh6VkGG0amSu2X6yhrb5EilzZwklVf6WV1rp6ysabJu2wDanpSTKrMLMyU3OdF+l8DBik4laLIkZNbhI0pMWTcciWOGZlmJNo7itUu27VkfRpAT3TAmK0WqWjpsLKy3nJWfYZ3CWW8GWjo7jS+eK1M+rW+26x0u8DLUg4/9LUOliRR2UVvn63LLqjKzpwiWZ6vtggWO7H9sW/UeGkeHcMw4tbW4TqrSa8AXnKNeaXm78ha2Dc0I21S+B8DGumLSUAuWb1E76AXlnWD7MewHdAXnhe6xv6blpdnn2FEBXxCUhOeX6nHLD8Oty7wMte9IuEoR/vstDRyYPy7fLo0dh5cRFQ0MS5Q5Aum5HXXWEKC7bqgBMDbfNaFYlW+SCRO65nHsf7dWyyIVVJtVoLCdwBPq2BXpbz4/a6S8bUyhKeR1avi+oNd4taJBhVqjvm8xhU1jBfaDHJmZbEYyQjIAi8sxLlD2NLZ5VK9Dtz3A3pEfVYMwRY1sxjNKndF2NShovEBt/nI1ZLgXjPGxWWlqKKTKdjUeaEpyuETXUBzMAQ5BLPDQlipTNoczoFUayGCsEo19paLejNN94bzSfPnItBK5UA3VYqW3qar4z1Rj4KyReXK0OoRHKw1Dm6tV+ZMJnKPGMEYtW3icpUbsiWrI4TiiuDEK5hdnyoz8dDVGUsxgLVfjAYcTwHM0XPjMzJFyvPJutdI4/BfgfZNKrNEH54PO4W8ME7L0nJvPj9HXDDWUC/Ve4d/V3TTwOZRBq/vuGlL0BzDS/qS64nAHtAndwh84bP9Yu8uCIj0hT38HbX9u1gg5bmiODFfDemx2itH8CUqH0OCk3FR17FSubK4yvoE2Lx1XZPx3utIrDs7RxSF65Tt0AWuvguZTO1Wes01GgIvGFMiPjh6nPJdljsouNYDBSOWjzx8xQt41vlgm5qaZTpielyHvVF124rDQuY/Tax2j15qmn6NT4H0ylPDt4YBS1Y8fnFISftf/+Of68sO+wRD2EHL0OKUfAh3sBd2bvajfOb5IvnvkGKPnHU3tctHYQrlkbJE19ENXEJBZXNEom9S5g99OUTvtfZOKzZY6Vf+GXgO+IEAyryjLuhmjG3A0IzOIacorOEKXjSs0nbZe7a4gyMP136+O5HsnDrUGVs2qYy7Sa1w0tmCPrmBMswszLHjKljrwHHbVYcIWBu6VeXX0Hl6G6ug1iBw3qlC6fc1OufLp1XLlU/p6eo1ct3S7ZS7OVwMaZ4/f4aDRhOPDT6yST+rvPqO//9RTq+TLz62TBbvqTPGXpCVb5JdsYF+BkGxQAXaDGm0fDV/jY/rvt1/aIE9sq7GGB5yb8USWsjocBxOJ6rSTQT9dFQ97OOIcEu0tb2mzYAlBKCLIX1AD+ntHjjbjG3LfqEbBwvJ6e8Er8A/0yl6Q15440SLPRJf7Avju1OG55sgSAd/R3GZZRAIpdcorfMZejV+YNdKcxqBs3OE42EhKGCSzVfbizDV3dO7JUBAAxOEcpExwYkmOfGX2KNvrkUxHh1qjS6uajCfY0olMH7yCUUww5ENTS8wJ7CtoEHW8OoaXqIFO1mVtbYsFMMnI4Bhy/k/PHCGX6veuKxz9CeiPYAqBGBww5DOBdIIc8AWZPQKD16sOwJkkyFGln5MZhC8WK93ivOHE4Tx+duZIy3SOVNneFyD64QsypPAGvEfG/xW9xtraJkkaPNiCKf8zuUTeq85lpuo2x1sb7iw6eg0ECkr2F4u2SEtHKOtBOURNa0jIYQhTKoFB8MyOWvn3xkppjyiP4BjK/Nh6gI1qMYbp3FWUmmjCqy9g/8d/bqiwbDCGdgDG98OXN1kkfPfragioETIzv+vaSIfjYIDyUpwwyte///JGOfvfi+WSB5bK2x9cJtct2WaKnYzhKerEUZKN48Y+jaf/a5GV9fE65Z5X5aevbLYSJM6H4ifD0lfDmOwIjmmZXvMbL663a1yk5z/vP4vlWy9tsGvzm0m5afIlNRLI8Dsc/QGWKZB5WabOH7wAT5yjvPHBx1bK7at3mLxn3Tm80dTeKY9uqZYL7n9tD0+c/5/X9Lil8ovFW6xcjvLrOQWZcsm4wvAVeg/2Vs1OSjRjO/Ialz+8XH6wcJMFUoYob55dmmdZFYejvwDdD1O5+68N5fKxJ1bK25QOoUkC3jhqbHdxxaRiKVCbCL6gmdm7lE7hB2gWWU65719X77IquMSEQRaoZNlBX0BlUlFaouQkJ9iWNx9+fKVcwDWU597+0DK1rcrNeSX7z36T6ao3HG9tOAU4eg2EU7U6hmzoH1QkUBFEeQIlPFcv2iwf+O8K+cBjK+SPy0JOXGTlAn+TUaS0lNIJQDnpXBWQKOu+gDWmq6pD9f2R18ChRflXqIFBWQals5RHORz9BTLjtCZ/tbzB+ICsIhlGIsc4ZpS/wQvQJcof+ud3QcMaSt9Q2HR2ZL84zkeJNbTbF0D7RKh/9MomeXBT1Z5r8O/9myrl+R21xheU9JFx51+Hoz+AOIcPMIDhhQr9Gzrc0Rj6l6AG9D1k0GDLrNyyaofJc34X0Czlc39To5isJGV2ZM4JLPYVlL0+sb1afqJ8wTUYF9cg4PjczlpZsKveSr6Hqi7an/M7HL0FAextygPPhJfOmPxXWwhaxDaiYopsYmiZUK08vb1GVtU07iXLlyg/PK90S2kpBjzLc4K1hr0F1hb64s515VY+zFg4d7WOhfW96Av0GRlGW2aRm25jc7x14c6io9eoVWFS0dL9+kYWWLNeASH4rL4QOJRc0AiEOvjzR+dbJ9WPTx9mTROGhSNhCDlq9ClL6gs2qnClbKI74Dy2qiDU/63ULlbr+xxvXTy9vdaa0wACKBif0B+ZxQc3V8r3F26S7yzYKI+po4jDFo2tDa2ypiZUtgrIVvZVOeNosublVTXQg3W+AWj8QdMnOkeS0R+aRiMe5wtH/wE6J1se8AJgvSFreDFMb1pZJt9buMECKKw7j/wdIMACzdKkg+UPqUMSbA1WX0Fwk3Gw9pprwJ+A88OflI1jnGckJXgAxdGvgL7Lmlpt3TMBEAA5Qos0aSJw8tvXtsp3Xtog1y3bZo4htBkJeIHqLZY5YDcRdNkfG4dGOYGjGD2WF3fU2fIIzDLsOJrAZdKkyvGWhVsLjl4DpVvf1nMzHIATyCJq1k1dNq5I/mdKiXxIX5+YPty2y/j8zJH2InNyIKBxDQLT4RhIECkmIoziJkMYDYImdJz7rRrEv1q8xUpEUe40n6ADHY0KWBtCKRGNPAJbmfUkrEvpC1D+6/V6GMTdAaODF2EZOrFSEuVw9BeQ0Rii0SCwiOymE/c1i7bIXWqwVra2W/Mu1u2y9RLNilhLRZMmmp2Fgn6D9suZIwBT2dJhRnY0GAs8iRNLFqWv64Qdjr6AOii6jDa0d29HEbj4/bLtVn7NMh6CKjQOZF0tfMFax5OH51j5dpbyAtRKxr2vdBvoLa7HNaKxtbF1z84AnBoexGl0vHXhT99xUMB6LAzcc0blybfmjZYbT50iN5822ZzFy+lAV5JjjUDo1LWtqbVHg7a3QPFT1udwDCRokEHWbl/ddikrTR+SYOVFBEmOH5ptneroVMg2NnSu++0JE+WT04ebwby/wPClnInIsMMx0MAhpJSuJ8AXZM8JMBI4oTnTO8cXW2Dxa7NHyY+PHifXnThRTh+RZ9Un+wv0xJsFOR2OWGC3imZKoWv2sXURjh/0TnUJ5Z9njMyVD0wKBdy/PX+0XHPcePma6o0D6eaJvmIpkKsKR2/hzqLjoCBHlf5nZoywxhln0BlSHUfLuLDX3K46+c+mSmtG88Vn18rZ9y22tSwOx1sBrJklc3jDKZPlqYtmy62nT5XvzB9jHR7PK82zMm0yjTQrOBBHjzImGjuxFsXhiGeQB6GJBwGTW0+fIg9fMEv+eMok2/+M5QpshUIWhWZP8ETkPnJ9BQGdduUNh+NQAAHFr88ZJfeeO0PuP3+mdcf+7MwRxhenDc+VmXkZZl9FNg/sK1ARreq5vr6nlsXh2DfcWXQcFKQnDpbzx+SroEuW2rZOuXPdLrnkwaXy4cdXyZfUQfzhyxvNWWTTcDb1p3TC4TjcAa2/Z0KRfGNeqTmFRItZB0I3YNqgP7uj1srxPv/sGrn61S3WxGB/1TfHuU3siHfYmtn0ZPnGnFJrzT8vvJcjZdRkI5dVNVjDqF8u3iIfenylPL6txvhif2EGsfOF4xAA2cLvHTlG3jG+WCbnpFu3eKqo1itfsJ0MSxronI2ueDRif2qHo7/hzqLjoIA1JaUZKZKakGCNPtjrkI3DMYbZS4jmN2yb0dix27YROJCyIofjUMFcNYTJkrAeMUGdRDYO/9HLm+THqvBZl/LrxVttg/e711fIyppGM6T7tvrE4Ti0QJOaC0fny5mj8mwtFs0+/rJqh3xv4Ub56atqCC/aItcu3Sa36mf3baz0ElLHWwJkFE8elmP7LA5NTzJ98AfVDTRH+1lYX7Ce8bY1O2yLsHba/DocMYI7i46DBDVySZno/3RljC6nwwhmbcpxQ7PktBG5kpfsbZgdhz/YbHxybpolNtju5cYVZdbs5uaVZXLXunK5d2OFreuiIGiE8od3nHMc7shWGieDQkt+SqYXlNfJ75Zss66oOIhsIfPQ5ipZXt1k39Ncg7XuDsfhjDHqLE7MSbN162TZ71hbLtcqT/x+2Ta5bfUO0xePb6u2LTMsOJ91YE0CHY6+wJ1Fx0EB3bXYg7FdHUWaFRxrm4on22axOIkIQTqkfmV2qW2nQVc7j4s5DnfQxIMXRu/2plZrKhAdSMlS45lN+I8rybGOjA7H4QyCinRWhNJpdMa2MezFGIkkNYYL1WhmP8aRGSl97gzscBxqSFEaD/ZLxI56ZGuV7ckYCbbIoCqLrsFUcjkcsYJLYMdBAdtqsJFreXOoBfq7JxTLUxfPlnvOnSF3nzPDFmtffex4mVOYaXvAdWc0OxyHG2iugaOIcUxjAkqN0obsnSV5z8Sh8rmZI+W0ETnhTxyOwxcEFhvbQ3sqsqE4Jdo0s4kE+7p9bNow+dPJk2RcVqoFXByOwxkNbZ1mRwH0xednjbTtMgIMUR6YlpsuV80fbbxRlOpLeRyxgzuLjoMCGhBcv3S73Lxyh7y0q94cwRHpKdb6eWJOqgk/Ghdct3SrfPjxlfLk9hqpV+GIwMNQoKzC4TjcQNdfykzpVEpX1O8fOUZuPnWK/PmUyXLzaVPknnOmy6dnDLdS1dbO12VXc7tn3B2HNQgo3rKqzLZ5YckC8v/HR4+1LsFsuXT7GVPl18dPkPdNGirDM5JlV8sbG5g7HIcrllU3ygK1nTY3tEri4MHWQfvqY8fJbWG+uOX0KfKbEybIxWMLrTfE1qiso8PRn3Bn8S2GR7ZUyTdfXG9NNp5Sh41W+/sCe1Rdt3SbfHfBBvn1a1vl6bKa8Dd7gwzKqpomW2/yi0Vb5PsLN8pVesyPXt4oP3mFhh6bbIH2rat2ypPbauSG5WXyA/2OTZn/u6V6z2LtV8sb1OnkehvVoNhhe9gFqGptt2MYP2PvCTiu1y3Z+saY9/Fbh6M/wabH/1xfLg9sJuveJlPUKTxrVJ5cpAr//NJ8NZSzrRTvznXlto4RHsJbHJqWbJ3wHI7DDWyp9NLOOrlhRZk8W1ZrHXzJLl44ukAuGlMgZ4zIsy7CVKCge9AZOJYEHAkuss0MZaoOx+EE7JZH1T67ccV2WVPbZJlEluxcqDxBQyj2IWWv6lfURqKzPF20mzt2G/9QsYVucTj6CwlXKcJ/v6VR19ZpDNgYLgM4XLGxvkUdvlp5XpU1DTferBSUbMeicIt/upqyvmRfQMEvr26UF/X8XIcXBgHZRhoW8D1XXKfKn9+8oK8V+jmZl+B4On2RjVlW1bjX88Cofn5H6LxbG3seB/fEOfaMeR+/PdRAie9Hpw2ztQuxwEOqvHhGhzNYJzJMnbNtSic4d9DM6prm8LfdgywhpXFlTW2yROmU9v7V3bT3xzAmk0LQg9I7+I/XZn3RHZhrPba1Wv6+Zpcs1L/hA1ql85vnlX7X17VIghrGmUlDzHjYoO/hG8YZgEYhcM9qPR/8gWENr0SDtS6cq6wxNGau+2bBokMFdJydV5QZfte/oHsnHWwPd7APIi4Z97tE6e2xrTX7zPBRHcKWSAmDBxv9Ive7208XWoU+of8mNXar29r1962qj0K8gf5YqPqC7TNuUUcxtJ5xkPESfLm4stHoFl7JT0mS1CGDZXlVk9y3qVJqw/SMYwm9w5N04oa34Ito0HgtI3GI8fKeMavuOByAU80+lrECQTHm+nAGGb9MpRd6VkNPT2+vVXp8c3qhSRM6GxvoGaWx1eoMQs/RoLIEOoRHsM/IMrKdDNdaoXzx0q46a3xDYzRsKPZbhI9Cv20xXoWWR2YmS0VLu223gS2EbgNZqkfQdzim8BGBe3iwOxSmJhl/YIc9o7porfJed2M+FHHisBxrsuXoPQa9rgj//ZYGzDbvjoXKrJ7ad8QvaB608LJ5Mdun8gvPrrX9zhyOeMb1J02Sj08bFn7Xv8CRmX/nwvA7hyM+QdOspy6eE37X/7ji0eW2NZDDEe/4xtxS+cFRY8PvHL2Bl6E6HA6Hw+FwOBwOh6ML3Fl0OBwOh8PhcDgcDkcXuLPocDgcDofD4XA4HI4ucGfR4XA4HA6Hw+FwOBxd4M6iw+FwOBwOh8PhcDi6wJ1Fh8PhcDgcDofD4XB0gTuLDofD4XA4HA6Hw+HoAncWHQ6Hw+FwOBwOh8PRBe4sDjBShwyW00fkyoWjC+SiMaHXBfr3MUOzJTtpSPhXvUNecqIelyWn6fnyUxJlaFqSHFWcJScPz5HEwYPCv3pzzCnMlBOG5ciE7NTwJ7EH4x2dmWJzE4txTMtLl3lFmeF3InN1DpiH4tSk8CdvYHh6spw1Mk+K9Lu+zKvD4XA4HA6Hw3EowZ3FAcKQQYMkS53B6eqk/PK48fK7EyfKb04IvX51/Hj5wVFjZL46L+lDEsJHvDkm5abJt+aNtnNN0b+PLMqS/51TKr84drxkJvb+PJ+eMUJ+fPRYuXhsYfiT2CNN7/usUXlyrd5Lf48Dh/1DU0rk8zNHhj8R+cIRI+UT04fLlLy08Cch5CQPMYf+hlMmy5yCTBun4+ADei1JS5KRGcl7XjjpzH9fkabPF8c+LyVRcO2TEwZbIIVzDemls88xhamJNqakAQ4QpOvcDNO5YI4SVI70FwiEME8Erfg7ReegQOeQVzBvjCXyGY3Yz2fkeHPwrHN1buGDyDnntT80maTPk2Nzws+XZwl98/x6S1fIPwJqjGsgwWihS/g8Q++jPwHfca0gmMsc8B59DhgL8wGPMr8j9MXfvZU1jr4B2mW+I/mBF/Ia3d5XQP/QNM8XNiDwzvu+0BX6gufOOQbyuXP/zEOx8nV/6grOnZmkskCvg54A/IuswtbtDjyzQA874h/uLA4QSpSJcFDuOGu6vFLRIJc/vEzm3bHQXm9/aJlMzU1Xh3GsnD86P3yEo79w6vBcFVrJUtHaHv5EzOhYV9ssz+2oC38SwmdnjpAfqSPt6F+8e2Kx3H3uDFl42bw9r0cuPEI+o/PfV5w2Ik/+ePIk+e780ZKiynNWfob8/cxp8qUjRklpRkr4V/sGgZvfnjBR7jpnhkxS3hxIUHnwwPkzbY4wZPoLo7NS5c6zp8unZgyXCdlpVqXwi+PH22tUeN7epmOJfEZPXzJHvnjEG0EXx8FDSXqSfGX2KOODyDl/6e3zLFDYV0xTOl6gx35aeWqsPmuqW+7S5/1lfX5FamC+GbCBzx6VJzedNlm+Ors0/OnAAIP8VydMMD7v7+Di5cp38ACyCCP53NI8+eXxE+TDqs8BY+E5/ee8mfZ8Xrx0rvErlTKOg4/SzFSjv0ie4PX7kyaZbu8rvqh64abTpliwODUhQX56zDh7/87xReFfvDnQF8+qLPz49GF7ZOVAgPu/Xufh1tOnmiPXX8Dpe/+koXKbXoeqOHC06ov/vu0Ik1vd4Sv6zP6mepj5dsQ/3FkcIFAqeoE6gsuqGuW6pdtkcWWD7GpusxdOyg9f3mQRM8owETwBvn/kGFNCvO5Rw/VXqqQm5fTeUDivNF/+dPLkPefg9U81ED4/q6uBd3xJ9l6/u3LGCBkZIfhQfl9WRo/8zTXHjZdjw8ICQxaHl88QVnz/FxW684uy5IyRefJDdboij+X1bTXoLxxTIJeowv/o1GEWmfrg5BI11CfYXDAn71WhRGYv8ri36TFBtPekYTlm9KAsuC6Owdfnlu6JeEWDqG9ywiBpbO+0aPsUNaIwApo6OqWtc7cUqOGEg4hjf86ofGnQ3zn6D1fNH2PPf3FFg3zwsZV7Xs/uqJVzdf6vPnZcnzIpPHeimJlEivU/aIiSbSLFCb08T9LgwRYBhaYHOkOQqvcDrZPR6M+hcB0qH5ZUNsqWxhabQwyfjXUt0qp88T9qHOOwPr+jTv4n/Iye3l5jBgp8PMDTdFgBAwy+OFfl9y8Wbd7DE994cb1srG9RnhhvcnJYWnL4iDcHfFAYzphA02QhoO+MxN5nFgm+5MJLSQNfYUGGFB6FbvsTZM+p+ClrajUan1WQIVl6/227d5vR/PFpw+R8dbz/s6nSntEPVJe/rsfx/AJD2nHggH4vn1Ak3ztytOrwJPnugo17+OJ3S7ZZdo/gLkF5aKO3gB+QdemJgy2zSMaY96l9qCJCXxSlJBmd9FbH9AeYA3giT19q4vQbuMY8tet2v/66dOqLpUNHDc2Sjt1Q/t5AH79H9QbLo7Ah+1L15hg4JFylCP/9lkZdW6f8cfl2aVQHob9BScPbxxWak/fXNbvk4S1V6pjsDn8r0qkMtqWhVapaO2RVTZOsV+OMb3HoTlWHic9fVUO6Xp2Wi8YWiPKmlLe0m2DAUUJ537exwoTb7MJMc6D+smqHHKPO37tUuJYqg2J4b21sVQX3uq3Xm6BjqWptlw1qeJC5mKwOE0xdqeflWmQWgrWDvB+eniLvnzzUHEqU5KvlDaYQjyzOMmFboce1qEH5qekjbB0lQmRldZOsUUeYc759XJFeI83+fk2NUcYyX49lHDhoS9WJHqfXo5Rjpc7B49tqVDm3ydF6rveps9ipF2McNW0dtraQ6OJOdbR36m9m5mfKF3SusnUemMdFFfU6h812fY4DnPvyCcWm1M8YkScT9br5KtzHZaXIWeqQzCnMUAGYKO06P9sb2+SDqnCYG5z7Nj0J47pvY6WNG8M5VsBJ+KgaIzi1scBDSpsv7Nw7u9pfQCHjfMAbq5VObl+9Ux7dWm00w6ta6RNj7BR1RqD9bTr38A00c/n4IqNbvuOFM7hCnzeAvqFLaAUDjlI7njvPc8GuemlWnoeG3qHnwBgPzsErJSHBggMY1ccprcNbGNKcj4DH0cV8NkTKm0P0DnhGH5oyTM4uzbOg0Inh9b+Llc4hP8ZzutIcx3MN/oYeTa+qQicAA58HYyD4Mkp5tkzpkM+v0Dkiu4lRDB1UqzzAEDiuJEfepzyJs8Zx0/MyLAgCD4Az9XoEVeYUZdpccd5GnT/4KhrwMNUN71QeuW7pVtmgMugI5YkT9Br3qmyp0zkh0o6R8JvXtspDm6vsGcFzM/MzLFP1l1U77X5jAZ5n5Jrj/gTy4E+qK2IF5hPa5HndpnN6x7pylWkNNt8b61pV1rbJycNyZbjKpGql8RXVjVYKeYLS69vGFFp1SkBLiWrEwjvoOQJxH546TJ5TXUCwEpmGPF+r5+UznjFLGT6m8iY4ntfsgkzJ0ueOTJ2uYyPjTAkaBjJZPX5zhP4GIzmgPUCm4W2qryL5dMigwVKj9AvnQOM4vDhU3OtpSq/IZZzh45XumANoODiWsSUq/SN/P6T3cbp+V6i8l6q0y/0jqwGBJ2R9cNxU5b9a1ffoHrIt79HvWKMOP3L+aco3O5QnCBZ2Y+vKhaMLzUl+dGuN3l+LnZtx8Exq9LxkajGAb1P59W/VEfDnOOWHk9Q4XrCrbs+4+hvoefRWrPDP9eUWWIoFsE2QMdhEyOQny2rlH2t37eELdAPPYJbSIXLsAZVPtcobOH7QLzo0oAd+g7zDzgJnq/6HF6DvZ/S8FykPYUNxby/p88POImCNzCGAzTlO1hc6pLKlw+hqjI7tvZOKZYuOA9mPXXaS8ii/wZ4JdAXnhY7RewFtzy1UftLxIJexP+Ab7DbsmvOUl+kngbznHPAbMj0YBy/4B/sJeUjgHV3FHOEww4/oKniNYB/3ik5BrpMB5DjAONFd8BSyBd4gUYBthm0XDYKIjIXvnimrMb3AOJlX5BUyJwDl7t+cN1r1dqrZnzyvB/X5xBLoZO7b0Xu4sxhGLJ3F6fnpJhgIWxEBg8EidRJiBObCGUKx4ODgJP3smPFmuP9+2XYzpHF+YFAyYZtVaSGo9uUszizIsCzZCzvq5OeLNstT22tkuRoWY1WYIVgG63gwzjGYceTIIPxSDcHbVu+QZhUS/AYBjbAjO4iziFL9g46H32CcI3Qmq4JvUSN0iY4dQcv6gfs3VcmvFm+Rx7ZVWxR7hhoZOMIoVIQ8Y8HAmKGKGmH25xVlFuk+Qsd8lyqh3y7Zaor72/PGmBL8+5qd8ks938vl9TqNg6wcCmzX8SAUEZJkaG9aWaavHXqfbziKAGGJUY0xQkSMCFhV2HhgDjvUuUUYb1dhj4JnvRvnwbElE4kwdWfx4ALj9afHjjNegL4f2FwZ+iKMzaroMLzy1XEj+ABv4MTBAyhv6L1Inx9OHc93W0ObPR+MhZ6cRZ5ntiq2j08broo2x8aAw4PzRkCDjAGOEjR2wrBs+x6lSZky/2Jkwn+b9FzwH5kZ1tp+/oiRxnfQDTTN9eFRaBvlT2SVcY/X62TrNZqVOAlA81uO5V4IViArUGpk+BaqY4ujeaLyGHSKIYHsIMgzLjvNDA6UIAYRgahZBelWXo2cwCkkwv4RNaqh+dD5h9h33F8AxgMfYnCg7MmacG8EuDDg+Y7ngCOAgwD//W3NrvDRoXKkSTnpKmeSjIcj5Vp/4nB2FnH2qAghMPKJp1bvpaOQy/ABBhjyG/mHkQud8Kx5jiPSU8yRxFmBbgiqYYhCw/tyFkuzUoymLlM+gP45lsAfdJCbMkRldq3REroAuoHW0D3B7+AFaKdCaX6G0vH71RHE8ITfoJPTR+Yaf2BcI2dx4qicgVfgY7Iy3JMoFb1zfLE5v2l6nxw7V581ugYdUakygb/hEb5HHqOXXthZa8bzh6eWqEObbUa+XVf5qVX5jWtyfbLgJ+nvMOoLUkKyBbmAvCBLgmMyUuXJycoP6EWam8GbtXpdDGz0JfPP7zkv971A+eJ5ncNdeu/IAfiI4M1/Vb8GBnl/43B2Fnlup+hzvHxikTy8pVruUocEWguAnsCm4/nxbB5UPYYsglYJOiB7cfhHZyLvMuz5rq5ptmNwkHpyFlfXNhmP4KTiIEHLo5VPCJicorQN3RFAQC9hHxFYzAjrgUnqHOF0oZOgCzKj0PEH9Hc4khwzRsczuzDD+GmNXh8HF9nGkgn4IzNxiOkLbBN+Tyk0PMff8B364lLlVwJGI3R80B2Bfmi/ZfdueVF1OWOi5Pzj04fben5kNeMgSLJZ55DqNsaJjoK2x+r9UZVD5RX3hvwLgB2E3CXQb3aT6iQcQPhknn6XqWPbGtZ7yC0Cq8fq/CHTCC41q0wjGOTOYvzDy1AHACgsDLU6FQRrVPgg0PYFmPXdE4otg4dzhSIHMB9Rfs6F4Yrw2Rf+rkbd559Zq87mNitLwPDl2g3tKLm9HZ6dTe3yojpJj6lywyj9mzpnZPeGqnLECLl0XKEpcpQfDiYCAucPxyIxYZAKtjdKY8l6rlXBh5AMfvel59bKj17eFBZqoSYAjIEsT09I0j8Uoz8AAPbhSURBVPMihF6rrDeDBqAUfrFoiwlynASiyAFYCxqpQCJBieMXnl0rlz64VBbomO9WRffxJ1fJRx5facqc+/rWi+vlGj03mVz+7elcjoMDggjDlTdQ0hi03QHlzTOiTJso6cWqyMk04zS98+FlcvEDS+TXr20154ZyVRb378uvzlJDAmeS4M0janRc+fQaO8cnlRYYBxmN8WoQAxbqc16U249f2SSf0N/cvaHCjHCMD2gTg5aoab0aHdA45+K3CYMGy3UnTjJlTnaHElKyfjivH31ilXxTaQ1ljEP3iBo2H9Nzcyyf49ARbSaLc8daAifbLLBxozpjP391s1UWvEsNIJxQgk/v++9y+crz69SJaLLIMdlBDFiQoteEZ96udP+eR5YbnUeCaPL/zi21uaOcDsMCQ+XPp0yWj6lDTbkX63BwRm9dtcOuFwmCGTjYtRHrfx0HBvQFwSHorjugPaD5zz6zxoJoGI1Xzhhuht4zqis+8sRK+dBjK+07DF1oYqwapW8GgiVkYW5ZVWa0yOt61R3oLYIk8A2OFIB3jR/Dv3t5V70ZnDhq0BDrv3B4qfLge+QuuuxINTTfrrSL0R2Ayo17lK8uf2SZ0X8oq5Esf1+7y469RI+F/8ke4ZTOUCP600+vNocXx/kvK3fIL1VeQ4u/PmGC5KsMuHrRZjsWGYGBiiHMeIAOz87/b+VF5op5RC8FOpGxwdc0jrv5tCnmpFB58uXZI42nqS5g/ScyBP7+lDr031uw0YIwPAsqH3CScSQw/h0HjpzkBDlOnz1VWDhAPK9ovKb08JNXNsu7Vc4RGBivThMZunNK8+SWlWXy3kdXmM5/Xo8nAPcJlXdvVq6Kg0i1FE7RD1/eaDT1wcdWmE2FfXK+0hSOI8A+wun707Lt9ruvPr/eAopWjgzNKN0ity9S3XHX+l2qB1bKZ55ZbXYWNH+Z2ljQJUBnJOsL2rpc7wf7hd+QubtSaZ/z8/rOSxvMDmQ9IDbSX5XHXqtqkHWqy9CbZEZxPj81Y4Q6ca2mezjuWpXj8AvLmrhHdB3nJnByp9pG7310uXx/4UarxIkEfMQxX59TasEWkgBUdX1u5kgLVA1T+v+cvqfhIktHcF5ZEsRyhciqA0f8w53FQwBEwDD2KIeLrAEnuvTktlqLzMKUYzJ7t5AaQ++qI8fIc5fOkYVvn2dZleiunkSXECaRQHlimKIAGQ8lCdFOJllHjNHI8+HURv8uACWgjIEXDQIC4dgdBkuogyzOW506uJFgvDi1CLgAW9S4qlLh/GYg8sgckklFwNs19LjI0glH/wOjkvI2nkV91PPtCTgvZLHvXFe+J+iCgrxZDUYivl9URUVQpCeQlfj3xgo5+q6X5Ro1KFcr/QKCGtADPBaAzDTGHhltnFYyk/eoIiULgZIk44IBrTaDBUIC/sGI/fOK7TZWDFTKbwDRVoIQQaSW6Op5/3lNPv/sWtkUzvaRlSF48fo+cnQoYjJHKPJ/rN1pBsnLahRwvnLlC8p681NCRhD3yz32dLZbV+2Ud6lBzRg43/+pgX6BjulrL6yXh7dUqnNbJUfesdCqBCgfigbRbzK3q/S7nkfs6AvIAhMHxEnrDZCBZNKQxS+pIYxRRibx6le3GK0SVDh5RE741z3j6lc3m3P1q8Vbw5+IOUDdBc3Iev85IttKNQZZCrIaOGMT9V8qPchEA3iV8z+pRuNcdbxYWxZgqRq3gfHP77794gYbR5DNxYB9QB27DWoA9wR4jY7WOLM4j2S9AOejWoJADdUDBiVU5odrRus8AJ/esW6XHHXny3KdGtVPK+//Qun/pHtelU88udqCn19Xp/Zt9y+R36rTHoCs7D/OnCZ/O2OqTM5NteDOa2rkOw4ctiZQ5QwVGej93gCHH8cdWv2VPieWNSDjcTbhEZYDpCXu2ySGRj7w3xUW7EAHgF1N7ZbZjO5lwLjuVzqlogRArwT3oEmC6cepvmC5D44fx2+ub7XAIKXLyNZQkCRk0xHQhy+wucCLeg8EI0/516t70Sz6iiBKd2sFAZlL7D/Ox1iw1cCzei/oUBITF48tkJGZIZ1JgIplBj0FOb67YKPNBeci64iumHfnQrlG+QPHlCAVgUkc8imqG5hjHN8/LS/bo2sdhwbcWRwAoJhYp4FTQhkPRnJ3oIyENRgsZMJYgP0jRQC2McIBucA5iO7uC0S9iAJdd9IkK6m5QRn2M8+skXs2lO9lFAOUamB8B+Atn3AtHDf+3vsXYkKK30XeUtRpDDS++fd5M219B+W2RMwond2pinlfgGC53+hzBrIxcgYw7tW9Db/bG5Se/OaECXLvuTPsb0qk7tG/bzp1sho4IYH5y+MmWFc7R+wAXXX3fLsD60IovyFwgOIJaIC1RmQaKSnCYSTY0hMwPKF9gg0EB8jsQZc3Kh0QIY4MPnTsDilisnoEP6B1DITyljb7Hdcapk4b56MELgg2YGxC4/yWoA4lTe16Lr6vVyUc8BnHMW5kw5eU7jA02TqGNSmhcrzuQYkQCphy1KZ25EGoFIh7wlCh3JDvQZNek897ml7mDmedLCcgIk1pI+fkGJxPeJSxRxskbPdDafdLasjQtOv13jxEx5si9OSVL+zfNwdZAYJ5PEcczECWswaRkjBKiKHBNwPPmHPAYzRFu/ucGfLd+WPkaDVyIwHdYnSz7CAAZXYEfILrUP2CwQkvBOD8ZNv0cCmJaMyzrbFNdkSUunEc45iam2adKeFPuvRSYtoTkpXnkQ8Y7BxPNhFAss+pYcz4qBIgwAqVwofQfrTOA9A+x8M3VM206O+4D+RLbkqCfQff8op0GKgQ+PZLG8yh/P7CTaZTWFNMuazjwIFMDMmY3skZSiApx4QeeE7QAstNkOnIOZbo9GSLBYCekLPQI4G/606aKP9Su+GW06ZYED0S6Iunt9cqjYdonusSROEcNEnCvqOJDnSDvIX2+K5Sz79G9RnrFoPMPeXN2I2MF+A0oid40SgGWwa++JnyR2FKz9tkDM+gXDXRKl8YS2D3MQdLKhvsOmQW0WfoOGg8mKvuEJIvBLSY15BuoW8E4yOI+7zqQcbNe7qQw8N/XLZddUqr3avj0IE7iwMAFCsRGxiSsi/KF7oDZUCsTyMSDMMi6CJ/SrkBWQXOQ2ODSEXcHSgL4FwoO9bfUZZ674YKWVXTbOUckcDApp49ElwPGWTCtqPDnNPo0leyCkTnKCXqDmQcKddh3QhHEnn73ZKtVk5LNA2F3RM4I2t00nVc0Q4ApW/MTU8ZzGggBHkGCGzKZBdX1Fvkm2u06Ngpz0PZB+Wujv4Hjj1lyNBzoCSjQRaYNaPQDiWk/K5dn1fISHuD5nBkUITQCQ7ovoAz9e15o+W7R44xAwCSplECRkGkQwRlsfYvkrT5E1pHObPGiqYzkGB1S8eeY6FJlDoOHDwMz/AVxkQkyMqRDWEcrLnFMKCUisY2EbfWBVybO2QeIn+GgmZeEsPfAwySffEI0e5PTR9uTTvoVMfaDrbCoLybUjvKD9mChHWZgXFEmWSoU2eBZUFZb7rcsqGOgwHohv/21eUTZwu+AawvhMZ4/oFxGYB1j3zU095nkUBGEyz7kj5/6IDMCJkPgiWRQHdQPRKZ+QyCKUEAk395z1KKSHBvjDFSj2BkBs4dH1NOC1+wToylBmRLeRGk6QnwvPGinlsvEYHXTd4zDziUBGwBhuvev3sDGO00IfmO0jjrssjMMKavzRll6znJYr5D/6XcjmxQALKwrMVHt5G1YW0Z+pcAsePAAM1Abzjv+3Lw0A/BEh1okF9GOynQGsECftPzmUJA5l0xqVhYakBpNXwHLZKtbA3TbAD4jKAE9hbgumTyoEmqBYJ1xvBp5IjgE+7N5Hp4QBbU1PNzToAORFchd9kXm3JXnLLlVU3Ge3trgjfAdU1nKh8yFs4L0JVcE52EzkQ+ME3R8iMaF6ueonyXZlTo0S8cMcr0F+t6ydx/duZIqxqgcRW6g3W9D22u3MPfjkMH7iwOAIi+0AwGhmGhMU5cZKQXJ+0cNcYuVQON73CCWAOC8iMqiaAI/Q7HK9e+39bQakpwX2BvR/BkWY2V0rFekpI4oqvRWUkymkRmiQgjsFiPNSYrxTIXRGYpnUCY0fCGZiIIPTpQUpNObT1txbsD94aDTJTvEXXIGAfdYCkV5IXB3RMQoqw9wMHDQAUIPoQQhi3Csred5liPhrOMM8gxf12z0xr1IPi5Bs05vrdw457yJUf/g+dLhJNni2HVHShTJkNOEwKMYpQcBkPIUH6DhqFnaAMFu7cq3hu24H5olhnFZNsJIFy/dLutAYNHg8gr4Oysu4jkFP7GAcQAIDoMT+tbyUt5YzNmvreotr7nfN1FVKH7Y9SQRPGfNjzXIsuU3TEWOhfv6x5Q6HzLPESOjXVubIOAgb5vlf8GWONJAwKaFjB+DA86OdLEh/kkE0TJLTKI9/A+BjNr5JAnNNJ6fNve6yAdBwb0BXIXY6w78MzZXgkHhs6pZFsIHPD8o51C5C8fvZkRyDo91lLRIIW1SI9srbLSy1tX0yxsbxlLpjL6OtA89ANPA2ie9+iGSMATfN4dTwD0Cg1J/keNTUpaKZdDLvMioNMToHgCi+goPX0EBtk8Mg/IhkgHtyeQlSdTOzM/XR3F9FAWSMdLcw9KzwnIkJUicEvmhGZrfAfPA3iIEj30M84pssFxYCBg/YLKRegOuyFYkx0JbCqqhujaTCAOmjQ5uTdBWFaM5wINvpmc5FqfmDZc3qcOI+f514Zyo8WfvbrZlsdEApagCRidcgG/D21hEdqWCzrardeETyNHxO9YjtE10PEGJqrdRkCP/hGMnyUHbKlDQ7+QExj+YRSoLAnpzMFmrwUVK5wDnh+itI4OC/j2zUBjIJrj4KxSdTJR/2XZA/PNfM4sSDe+4LP5xZlms/J7HF1KXgnyY7thEzIGR/zCn84AgZT/K+X15oR9Y26pdTsNHCa6pt106hTbu4pOoVcv2mLODOUKrIOgkxS/o1sdXdxwfHi9GWpa1ZlU2VCcGroOr09OV+OU7KYKCZR2ILTobIcThjKkLIH26XSpw7G6QR0pHC3WU504LNsWYsPwNE84Qo0VMpw4t90BGYSCRkYFTXaIQF2mAh0DFaFvxoe+iP4itIjsouD1I1tfgINK10iOxYglyoeDgcCkUU1vgZNNl7JqnVcyjWREiRwS/erJeHH0H3jerG3guY5XZRhdKgcd0Ezii+rYfWf+aIuKYkhnqUMEH0EfAMeRTcVRRGSLMQp7AmupaNaEAqUpzGeeXmNdWHHecIYCfgAoeZxYAiiBMYyRQWCFkjvWMOLschx8w3gBtIvxSKdGSuygr2hAvzRg4J4wPhjHE9tqzAEMztMTiIrjEMI7RKsZJ+PjnDhzRLejMzo9gcYirK0igEOzqkseWCqXPbRUHtpSaV0HL3pgib1u0t+QLb1kTIE9C57bt17cYJkUx8EFuoJyYowreCPS2IUGee601Ke0nw6M7UpDFSrraVgRMgBD2Read5BlQaby7HoCshkZi0xfXd0kX39hg1z51GpbG4WxR3YiEtA7PMErAHKUzo0Y9UhSOodyfYIRAeBvaBQjBL7pDtzpWB0HzhZNar6qPMpn3DMbpvcEeB7eZ67g2SAIydTRGAX5gOyAZ98MVBg8qPRPQyieA+vd4IEvPrvWxk2jnk8+uVo+9sQqaz7HHsBkHdFNAIeGeyXQBB9S6uc4MFBtwbr0xvbdlvVmyUCkswFN4ijS7Rf7iv2RoQUcd+iU54GjBG9QEooNAj3goO0LVDXBG3SU/5I+f8Zg6yeVnuCxSHB+HKWgaQ7XZC0x44CXCExSqYL+YCkBY+E7aHtidmh7l8hgZSTgYxw0Kk/Y45Z1hZyPa2A/9QT0D80LLVuuc8Y8AeQKNhVVbqxJDlXqvDm47k0r6BLfIL9estWaUrHunfWK2KzoD5rosEyE8lhsRJZW8CJLz3X5lwxpTwFiR3zAncUBAsxFdykW/6JUbj99qiy8bJ69/nnOdFsn9OXn1spd60Jt6WmCcdq/Flk06qdHj7Pf3XHWNCv3+vErm7t0NewO312w0SKz/6MGRXAtFlojZDAuEEBBpGlDfbNFX9lE/4VL58qHpwyTf6kh+bUX1lnZBOP/7oINtj3Bt9RZe+6SObbOiy6WdAb718YKO080atraTcBgCBMZYwyPvW22XDl9uGVQOC8CBAeV5iGsJaAL4+1nTDWHmnugO9clYwvs2AcvmGmR3P99Yb1lNfqCQFEsrWoyAYmg5bpkdZhnR2zBWgeMQUra6Kb22Vkjwt+EwF5qdOrsUPq7/OFl1mwAowEj4e1jC/coawIqGM2UgZEN6ckQDQDN4/QFx5NpJxJKphGaCIAip2sv/EM3RDLv7HNKNoduvGTcGRO2IB3faF0OyLazpxU8TPdT6Ksn4ORRphaUWVNJ8P5JoRbsPYEM6HY1PuYXhvbjwtgg0EMpEIbIHSpDetPoKQAZEYyJwGBgPBjbzZ178wRt2/9H5cJ/t1WrwbKiVwErR9/BGlDkJYGtl94+18ogA5Bpv+a48Vahgux8VB36BjUyX1RZyhZGlE+OTE+xLR7o3kl2DEO3t9lfDOvSrDfWE1J+THlqNAg4UsodgEwnFSBkm/EWaQKDQQ5dAuiZ8RDspOkNmfx9AT4MSkaR26y931fre3iN5lEY23RLvnhcgX3OdbkHHLa+LDHIUPpnPz6CiEHghXPBJzTywSkGGP+UYtP9FGcb0Hzqi0eMsK6ti8obrGTRceAgO76gvM466hLMZr10ACqyyLohA5/YXi13qgzEqcdpYUsHSiORc2ydgQOHHYANhfP5ZkAUE/QIZDTbwtx2xjTjsUjgRJLVHJsdogP0BWvRoUmy89hi6Av0A2X+2F/YgheMzrds/qNbqyxI0hPgg0i9xf1/TnXmCB0H33WHjWrXraxptEAO1TRB9pvy7rePLzSdScCwL51KaRYFj6+tabYlGQRrCXoShA9wldptx9/9isy7Y+Ge1z/0mbD0CP1ER1b0vyN+Meh170JggDkgYKJLsQSGLu35iToGIKlFpgKlEx0BRvkH0VmiYBiBdL0jWwfjk53AycNwQFhhTCDUyMih6DA0iEgFCGroMYSJOC9Uxc3aJLIgDCmylTTXIVobAEWJAYBzF4AyNRwvsqBErdjbkcgqn0Ua7bSNJqrMbyBAxsb3nJOZwADmnsjIcL8Yrji1GAGMn3UiQbQYcH9ck3umJAgBTlSc0tSeonPME4oCIxdBhZDj3FyLiHN3DiORde6XbmFE/Hk+scxC8vxwkoP1Sf0NthehVDiWQNHTaIbIMPQUAFrGwMOwRNFCt5RGs30GLbz5LU8C2mDPLIISZMco56ZMkmdKBmC2nvf6kybZnp90TqVT6M+OHW80AK1Y1lBpFkP3nNJ8uXnFDttfi02WUeb8Dr6BZ6AhyuFuWbXD6AH6wHD+xtzRVopNSQ8KnbH+YGGo9fhZI3PlA5NL9HP97b8WqfH5uh13hn7+yenDjb6gW6LgXIsI82dmDreyVJpSYYT/8KixppAp5SYTypgpBw3mDPpg/Scd+4h+w1vfO3KM7SVJafXPXt1ix3cH5uq0EXmyrKpBvvPSRrtvNlJHHtGaHWDcfEp/R6aXroKUpUeerUyv95vXtsnaulC5fX+D58lWH7EAxv78OxeG38UGlP1iDPJsyEwHZWLIQLLqrBNl/TeddylxQ45dOX3EnvVxlGUSBGHt3N3rK+x3BCKev3SuZewItJGd+aoakFRoUJJPWSuGK6V0QcdejG06KlJK9vul26RJaRSDGJ5jvVSQmYH+yC6QgWadI50QPzK1xLbiQL4i4ylVY0smxo7DNU91259OnmSbeN+gtE6gkEAO20YR/KEhTdBZMnSPacan/91WJV9+bp386Oixth4eeiOw+bNXN5luxTGghJVjuS7ZVq5734ZK+/uPek1A8JY57A4EiT6gTudZo/Kt8yOBTTp5X3PcONvKAxkAv3Lf6J7vHznW5ANNfJARBCFt7jeE9iLc19r8gwW6fz518Zzwu/7HFY8ut2cZK+AkkblDRpOdwp4I9AU2AzxCIAIaxJ7AgSJYgeyiOotN+HlG0CyyG5pfp5/RRIm1pThrbL1BlRe6h3uj7PTdE4bKB6YMNWeI0lOOx0lif0Sqquh8vb62RR66cFZo7bv+DhsBGU8gjy03CP5ALxxzxcShJrdpOsMY9X+TMTevLFO902y68GPq+BJopNwVOwcdgUz+xrxS4y/OhQ2ZotdgeySqCj799Bqzu96j5//E9GFW7fWdBRus6yqVCAT74DvGRknqrpY2o21+R8D1Pcp39MIgodETcHA5N13A2UIEXUXWELuO8dPtuCf89oSJlhUmcPW5Z9aEP40NyDZTieHoPdxZDGOgnEWHoy94KziLgA3hiRhH3icKGSM32qDDMMZBw5hU+6HL7zBKOReOPQqXEmvWTBBlxXnDaXqbOpwYG0RJMTJwBl+pqLfgDFmRSjUCJqlhEpTKBE2WUPB0fMMADDJxGAQY0GRlMEbIYqC02YYCwwLFinHAWhX2KgwMbKLbGOyMF4MBrFGnlyg0hsFK/ZeW6TiGrGuk9AkDGMcZB4JuczjawbGUOmGwL9WxcQUyQnRsxbjgmJ4EPyW0OKScm2wpGX3eU0XwdFlI+bPOk4xqdAfAAGzfQXfjoBNef+NwdxYBmUXWqPMsgvIxAPncZ+32m/bQICTAcyRrEQQX8S95JgRc+B0GJ4ZosCk/7/falF+vQ7AuyJABgg8Y6ThEVLVgHGKwU/5McI9jAEY78xRslQEoe5uj44mkGZw6DGpomuZo0Dn0/kp5gwUhuA/4lSBHsOUM98FyCJxYSlEJhPxnU4XdLzRJwBGjn2wSoJsw/AY4FgP5KaVjsiiUiOMQA0qou9sOBhCIhbcwwtlsn/tjGwacFHgYAz4A88M1KVek2yVApvCMcIBjVbVyuDuLAZD/OIEETSJB1ckipWv6EAQgcAe9Uo0UIPp3ZJ6RxTiTzyidoBvYCB8ZTxCewAF0SrAPsISFIAqfI1PJHtN8CScNuQ/N8FucMpbusLQoyEQzHngJ2iWoAOi0+vwOldtK4ziB8A3ZUK7xnH4eBBrgNfQMvIPMRxfhvCJv4TH0H/KXdbTcE2CfXvgL55c1yVa2qscSYEHPwY+AgCpr0wmeord6AvdG4IilIGQIAbTPml4qIvZlT+Pkl6qzSRXbQ5tjuym/O4t9hzuLYWBYHXXXy5YN8BlxxCveKs6iw9EXvBWcRYejL3irOIsOR1/hzmLf8UZ48i0OMhIY4JS1OBzxCCgz1uRp1wz96XA4FPBgkL11OBwhuK5wOA5fuLMYBhPBIvaeFgY7HAMNaJM1SrGkUEpcgoX8Docj1BCJLUmCxhIOhyO0NzN7VzocjsMPztlhDFZDvCgt0YxjhyMeQTMkmvfE0kZlDV/k+iiHI97AFjzBep9YgDU+rIvzuKIjXgE/xHorAvRE9D6aDke8gXWarLF29A3O2WHQRZOFxllORI44Bcqf5ijB9iaxQEFqkjWYcDjiFcPTU6wVfKxAQJHmJh5EccQrMIjpoBtLoCeCZkoOR7yC5j90f3X0Da7twkDxn1iSY50MHY54BN066UIZy/K3kWpwsD2FwxGvGJWZLDnJsQvyUQpO90W2F3I44hE5agzTnTOWYGuWWF/T4egrhqUlu52/H3BnMQyixScNy7X24bS7dzjiCWyzQAt79g2LZRMm9pPECHA44hGIarYoiGXJXVriYJlXmBHT0leHo7dAOxSkDIm53CaoSHDR4YhXYDoRXCz2DHif4douDIwOuqEeq8Y4G/g6HPEE9idjo3TKfGK5ZnF8VpptRO9wxBvIsNNo5uThOXv294sFKHk9eXhuTEtfHY7egkYzyOxjS7LDn8QGLJEY57aTI06B3YSNf8zQLJnkNk2f4c5iBCCmt43ON+ODmn+HIx7AhtCnqHF6qtJlrDswkkWZnp9um9h7p2BHPIFN2c8YmWt7j9JwJlaADyhjgifGZYU2inc44gXT8zJkXlGmlKQlhT+JDTKUH+cUZshpqqscjngDfUmozJqm/IFN5egb3FmMwvjsNDMCThvhAs8xsMAvZC3tScNy5MRh2TJqAMpBcU4nqzF++YRiK9FOcofREQegLLs0I0WumDjU6DKWgQyuxFKFS8cWytHFWZLlgUVHHAC6JGhyyvAcpcvsmHd2R1ccUZApl44rtCYiHlx0xAugxaLURHm7yuzx2am+1Gw/4M5iFJISBskxQ7PNOA4MgVhncxwOZBmKf35xplw2vkhmFWQMmIBD8Z86IlfOH50vRfp3LLuxOhzRQB6Pyki2rOLpSpdZSQPTaGZuUaacNSpPDeQMN4wdAwqoD+fwuJJs44lJOQOT8R6pfEmg/Rzlixy1nZwtHAMN7JXi1CQ5Y0Se8UahN7fZLyRcpQj/7QiD/exGZKTYVhpra5qlvr1T2nbvlt2vh3/gcPQT8MMol8hOHiLT8tLlS0eMNOVbpMJuIEGGc3ZhpiypbJSdzW3S1vm6ODs4Yg2cMvYaPX90gXxmxgjrEDxoAIMXlu3Xyy9Wvmjq6JTXnSkcMQbGMPvGjclKlW/PGyNHFWcN6D5yrAubUZAhC8vrpaq1Q9rVcHK2cAwECCyyD+9Jw3PkG3NHm7z2rOL+wZ3FHpCoBntJepJcoEYJ2NHUJuUt7fa3w9FfQMnPVEX7nolDzVGcX5RlyncgDWKAQcI4ZhVkSqdaxJsbWqSurTP8rcMRG4zOTJVPq5P44akl1tRmoLPcZHMoh52UmyZLqxqlrr1TOjyq6Igh8lISrfLjF8eNt4AejZcGki0S1BinImt+YZbUt3fItsZWaVC+cDhiDbZyef/kofKFWSOt/JRgo7uK+4dBryvCfzt6wPq6ZllZ3STLqhtlaWWjLKlq2JNxdDgOBKy9YrH1iPRkGZ+VKrMKM2RWfoZMVuOTTZXjbePv1s7dsrG+RZ7eXiP3bKiQBzdXmfPocPQXQutNkuTEkmw5tzTflgkQIY6X9bM4hzVtHbJMncW71pXL/ZsqZZ3qDIejP0EF1Dx1Dk+nvG5krswuyLTgRbwkTlpUV6ytbZbHt1XLv9ZXyH/1X4ejv0E2MTspQU4YliPnjMqX41Vv4CjGeg3v4QZ3FnsJBF95c7tsVCNgY0OLbG9ok+ZOdxZ7wvLly+WOO+60v9/73vfK2LFj7G/H3sAQJmNXkJJom8WOzkqx+vqBLCPqDXY2tVmZ0cv6WlLZIOvUKNiun9W0dphD6XDsLzB2KcWm4yjrZek4yrpAWp7PyA91sovXdeQLd4V4YlFFvayobpJN9S1S2dIuTR27PajiOCBQPpeZNMTWXA1LTzJeOLIoS+aqwzhW9QaBx3jEloZWeXlXnfJFgwXaN6gNVaa6oratw5YzOBwHAjRBEHTHdsKGmqm8cUxxlvV6INDoa8oPHO4sOvoFd9xxh7zjHe+wvx955BE5/fTT7W/H4YdnymrkFTUEiCLvaG6T5o5O8UUq3ePZ556V6uoaGTlihMyaNSv8qSMSlJaSNRmekWwbi8/Iy5AZBemSF8ON9w8UZN+fK6u10lTK8OrUMPby1O6xcdMmWbp0qZXan3DCCZKVmRn+xhGJZOUJAig0kaEK5Ug1huOx+qQnQP9Pbq+xQMr6uhbZpbqipcMDi92hvaNDnn76aWlqapJx48bJlMmTw984ooHcQF+wfn206gv2+5xTlGlrFb055cGDO4uOfoE7iw5HVxx77LHy/PPPywc/+EG58cYbw586HG9dXHvttXLllVdKYmKiLFy4UGbOnBn+xuF4a6KmpkbmzZsn69atk69+9avyk5/8JPyNwzEw8CJeh8PhcDgcDofD4XB0gTuLDofD4XA4HA6Hw+HoAncWHQ6Hw+FwOBwOh8PRBe4sOhwOh8PhcDgcDoejC9xZdDgcDofD4XA4HA5HF7iz6HA4HA6Hw+FwOByOLnBn0eFwOBwOh8PhcDgcXeDOosPhcDgcDofD4XA4usCdRYfD4XA4HA6Hw+FwdIE7iw6Hw+FwOBwOh8Ph6AJ3Fh0Oh8PhcDgcDofD0QXuLDocDofD4XA4HA6HowvcWXQ4HA6Hw+FwOBwORxe4s+hwOBwOh8PhcDgcji5wZ9HhcDgcDofD4XA4HF3gzqLD4XA4HA6Hw+FwOLrAnUWHw+FwOBwOh8PhcHTBoNcV4b8d+0DH7telqqVd1tQ2y+aGFmnu2C1NHZ362i1t+p1jbyx7+SX5+x+utb8/+PmvyNgp0+xvx95ITRgs2UlDJD9liOQlJ8rUvHT9d4gkDB4U/kX8AsnRtnu3rFWe2FDfIpXKHy3KDw3tndLauVt2h3/neAN//O7XZcva1TLnhFPk4o9+KvypIxIJSvqZiQmSl5Io+coTozJTZFxWiiQprxwKaFd9UN7cZnyxSXVFW+fr0qi6Ap3Bd4698dKjD8p9t9wgCQkJ8onv/UyKR5WGv3FEIm3IYMkxXaF8oa9pqiuykhKUX+JfV0D2LZ2dsqamWTaqrqhuVV2hOgJdAX+4rtgbLY2Ncv23vyJVu3bKCeddJGe+64rwN45ooBUywvoC22l4erJMzk0zfRH/nHHowJ3FfYCJ6VQpt72xVZV+qyyvapQXdtTKa/pvQ3uH1LR2Sl1bhwk9RxQa60R2bQ39PVSVf2p66G/HXsAoLkpNkuEZyTI8LUlOHJYj0/MzZKwax8NU6MUjOlVk1LZ1yhY1hDfVtcjzyhOvVjTY+0Y1iHEaMYz5nSMKa18TaaoXyS0SGTkh/KEjEkMGDzIn0XhCeWCGGsXHDM2WcdmpMkb5IiUOnUYonYDiFtUTm9UYXlLZIC/uqrN/4YVq1RP1yjMEURxRqCwT2bZe/1DTbuIskRTXFd2BoGKx6gh4YqS+Th6eo0ZxuozNTpFi1SHxCHRAVUuH6YYNqiueVV2xuKJeyprazFGsau1Q/ug0Z9IRgc4OkTWLRdpaRAqHi5SMDn/hiAZxdQLt2EuBo3hCSY7qixTVF6lmYzkOHO4s9gBmpXX3btmpQu3va3fJvzZUmJAjk+hw9CeOVsP4vROL5fIJxRYxw3iOlwhZoPxf3Fknt6/ZKQ9vqVLHscOCKg5Hf6EgJVEuGVcon5w+XCao04jDODhOMipB1mRHY5vcsmqH/HtTpayobjQn0eHoT5wyPFc+MHmoXDimQDKGJFhFSjzpiormdnlie438Q22o/26tUgdxt/KL6wpH/yJlyGC5ZGyhfGzqMJlTmCmp+v5QyMDHM9xZ7AFEgRdVNsj3F2yUpdUNZiBTcuez5ehvJKshXJSaKPOLsuQbc0fLpNxUSVdDIB5ACdGNK8rkznW7ZHNDqxnErvwd/Q0UPWV449VR/IgaAG8bUyjD0uMjm1Ld2mHZ9e8v3Cjr61qkpq1D2jtVV4S/dzj6CxjFJVSklOTIV+eUWkUK+mOgQQBlfV2zXLtkq/xnU6Vsa2yzgIrHFB2xAG5hemKCjM5MkXdNKLbgO8sZHPuPhKsU4b8dYaD8iYZdvWiLLNhVZ+99rYkjViAi29S+W3Y2t8mq2iYZkZ4sBeo8DmT5HfS/VZ3DX7+21bLs69VpZH2ic4UjFoDOWBteo7J4c2Or0uNuGUeJUdKQ0A8GCBUt7fLg5ir5rRrFlGLXtXdYOarDEQtAa5Rz7mxul9U1TVZ2l5s8ZEDX97IsZ11ts/zs1c3y0JZqK0HlM+cKRyyBvmC5zDbVFwTwJuWkmQMZLxUphxrcWYwCBPbCzjr565qd8oAaATSx8WIiR6wBzTWrgt1S32r/Dk1LlpEZKQNSSoHziuN6++qdcse6cosYe/DEMRDoUFpkTWxlS4dQcDenINOU/0Do/zbly8e31cjf1u6yf90gdgwEEMU0UKLSAxocmZmi+iJpQIxiliPQ3+E21RV3qq7AUIdnHY6BALYLyZ5dze1m20/PS7cKlUHuMPYZ8dcpYICxtaFFHtlSJf/dWm3NCFzMOQYK6FiUP5m8BzZXyqb6lvA3sUUdJdnlDXLrqp3WvMMzJ46BBIEKGsfctnqHZfNokDEQoCT7fuXLZ8pqvHGNY0CBSCbD+I+1O+VRtV+2qpM2EKBhzQLWsytv7mpuM2Pd4RhIYK+Q6Wb5DL0WatSecfQdnlmMAHLtHjXM71lfLitrmsKfHjzQtSkjKUEyE4dI0uDB5ojGszANrRNKsC5slLXE+3h7CyJLWXpPqQkJ0rL7wIy8TH2eWfo8aULT2tk/c4Owa+l43WhnflFmTKNi3BHNOm5V5f/4tmqLzh1sJOrcUR7CPCYqX3C/8U5lNB6Chhjv4eAowOuUdNI5bgiySfl8f8k5SZ+nnUvplewGs3OwyYbzMe+Vapyytjc3JTGmjT0Qg1Sf3Lex0tYp9hcoKbTSKb27eM/mIwNzkhJtvAz1cMgoQcvpKpfgd3idZ7C/d8UyAngijeepfNEfz5NzNquuKExNlJn56THXFS+X18tfV++Up8tqDxrPG10lh+gKmXQo2yA8DWgJ+YgNCA4Hmwo+YeuKFLUXUYcHck80o8HmxPZkW5WDMTucA9uFwDdbztA1NYascVjAM4thINiIzD26tVqWVR98RxHkqsD7/pFj5ZELZ8mvjh8vx5dkh7+JT4zMSJaPTRsmz1wyR246dbKcOiI3/M2hjfdNLpGHLpgld58z3TrIHQh4no9dNFt+eNTY8Cf9g2XqsD2rCphyiljqFgzylcoPNCnoL+OPbmXM41MXzza+oElDvOPLs0fJkzbeCebsHuqgWczVx44zWv7JMWPlyOKs8Dd9x1FDs+UPJ0+SBW+fJ5+bNdKa0vQHWI9y38YK28+wSWV3rIAhVN/eYVnFdXXN4U8PPiCr28+YKg9feIR8bPqw8KfxiwnZaXL3uTPk+UvnWnfOwwHQ8s+OUb542xHyM+UPmsnsDwjGnFOaL7ecNkX+dc4M+fSM4eFvDj5eraiXl3bWSUVLR/iT2IA17K9VNsh/t1WHPzk4oHTwHp2zF5Suzh6VF/700AQNib4+t1QeOH+mXH/SRDlleE74m0MbdHB/SeX9fcr/xwzdf90BLh1bpPMzS/6rco9y6oMFlg1g369Qe4a15Y6+wZ3FMDCKny6rUcOjyZzG/gDKn83XiWoUpibZhuzxDIzg7OQh5jSyvxMZucMBWYkJ9gzYl+dA13WwOTINaPi3P4Ggo7To1fL6mEYi19Q0yUK9Jo08+uuydO+DL0ZkpBhfkNmKd5DxGZHOeMloHfrOIi33aaJkzZSUlg+koyIZFPZ9o/sc89RfzjRdeBtVVmMc72hqC3/a/yA6zTKFjXUt/byV0iBbq4ysItIe70hKGCTDVE+gL8ieHA6Alo0v9J7gC/hkf5Gu+hPjl8BMjvJFfwFbhhLppeq4xRLLqhpkUUWDlaIeTFDVxJyNUv0QL13B9xfoCjJw2B7sr5x6iN9PAPgEvue+yAweCJAdPG/22SWrfLCA+UIPkqVKpyyncfQNh4f1fxCAgKWxTXlzuxkhDke8oVJpk+1cYuksUmJHGaqvU3TEK5ZWNloDpliBYCJldjROcF3hiEeUNbbJcpXbscTKmmbL8ruucMQzViudbmkYmDW9hzLcWQyDWv/lVY1SH8NyJoejLyBiu6yq6aCtBekNtje2umB1xDUoK2Lz71iBhjqLB7CxjsPxZqC5zCo1imMJsjXb1Ul1OOIZLB1wOu073FkMg2jYBhV2pKkHCpR+UW5ESSMlL7woWWBRPAuIoxPyvKdEI0u/53fBMRzPeaJLycjoUzPPd3nJe/+ecjEadrC24kAS/zTICJ0rwUoT+Du4nzz9m+85f/S98i/XDxZ9R4JSBBY7c67oMVOy0NN4OYZyny7XiJqXAJyHa9n8hI/hxd981t0ziCWqW9tlZU1jTLMZlJ/ujGGJXzSgWcqfI5+jPROlX559d2UqVBZDX9HPnmPgpWga5xQ0T4i+Bn9DY3x3IOUwVs6t9MM5oT3oEl4IrsM14BWuQQlPwDPB9bnP7viSEmr4iXMF4+Zf7oPzMQ/R4BqcLzgmuEaIL3u+R86Xo/cQXIfXvp5BLLGmVp1F5Y1YwfaRU4Ojvxpa9QbMOSV5PLvgeQTPEnrtCXwX0FdwHO+hyejnCN3yfKN/z9/QAsccyKM3HtXzwBucK1Lucg3e8xtoP/pe+Rte4btoIKcjz8WLv5H9nK+7IXNMiJfeOIZrQPc9GUnwX3c6JpjP7sYWK5Sr3GZJTSxBdr+iJb6M8CH6DHjuPNfIZxTIyO6WoRhvhek+oIXgGJ5rJDgaPuEa/D64Bi9ojmOM5g6AFBgn5w7GzLUi7wVaZwwBv0Z+xzHdlYXqT21cHBs9Zs7PuaLBR5H6Kfi92acJPd8gcxwtdyLH1s2l+hUsHyhr8gB4XzHoddreOWw/oFP/tUg29OMecqxv+sVx4+XMkXmyYFe9XL90mzUOCXDx2EJ5z8RimZCdagwLD1EW+9KuOvnn+nJ5envNXt0oMaJPKMmRS8cVytzCTGNAvqb5wqqaJrl55Q65P+L8MOl5o/PlvNJ8GZOZar+HUZs7dkuVGltE6NlIly0aKMtlHFdMGipfOmKkLK5skGsWbZG71pWHz9Y9rtH7Yyxc/5nttfI/U0pkeHqSCstBqsDa5MUddfLNlzbIOaPy5KIxhTK7IEMFRoItOGa9w62rdth6oEhMyU2Ts/T3F4wu0HGnmPBhzDA8pcM/WLix27VDX9RxnzEiT0ZmJlvn0+AarBuZlZ9h5WTz7lgotW2hNRYIrsk5afKVOaUyUe+d9Zo8A7LNNHm5Tp/XK+X1e61pvfX0qTafD26ulHc/sjz8af+AsUzSuaBxCEohFvjmi+vlV69ttbVh/YUTh+XIR6aWyNuUHp5SGv/yc+us9BVAo/DEmfocx2SH1qwQ2MGJfWxbte3lxTOJBAqQZkycc7TSS4o+e0p32Z/vlYp6+dkrm2W7OsCsAwWsHfn4tGEytyjT+IJAB1wGD5SpXHhE6RE+gnYC/OaECfL+SSW2zvmSB5YqX/a8du3YodnymZkj5KiiLPn4U6uM9uD1oFkG/HbDijLLVsE78NwI5RnoHAPsBeWZ3yzZKjsadczh66DMGfeXjhglRw3NlEKdJ9Z64shwvluU96FJSiUjQbOId4wvktN1PpFHgGAA8o8GXEcWZ8rDW6rkD8u278WHPIPLxhWFN8IP0R7rBPnN39bslGVVjXu61p2h8u0bc0vtuf5y8RaVQ2WypLL/SuKQYdedOMmaccUC0MFp975qaxf7s+SO57/wsnm2DugPy7bJd1RuBoCGaJhCww9oHBDo3NrQKk8qD/3w5U32WSSYp2/OG206g3WQwRp0gqQPKH2zPU/kczquJFvOHZVvDStK9PcYqoE8pNyQ67BFAjoKzFJZfudZ023t0iefWm2t6nsCY0Gef3ByienbZ3fUyJHKH5NVvsG/OOJU+kA7dCa/VGnvAr1f1g4y4xtVT/997S6l8aouWwoFvD8lJ31PcAg+WKS8zz6x8Gx9RPt85vkEvdeLxxbovWbbetvgGswNOmem3ht7aX7h2bV71jvxXOCL0/V6rKdDf6Cfdylf/HNDuW15RMt+wBgun1BkcgCZdse6XfK159fbd/0BntMJyn9PXjQ79EEM8LEnV8ltqr8P9jpemm399YypNscf+O8K60LcW6Avvzp7lEzNTTfHhqAAfAL9/p/Sz2P6TKmeiQQN19DpZ4wMPVfAPpbsefzI1iqzgwKgh49Q2sDOQTbS2wE+QS7UqV2xTWnhjypL0WvYGTibvzh+vJyv58emuFZtirvVttsXLp9QbLIN2c78njo8V+brnLAGlrnmPFcv2mw6DtmOfUnHfSofCGqxXzi2ZiSQ/ScpfVymv2duCJQwZjavp3z5N6rzF6qNGgnm723Ks9Ax69FxNmuVr55XGwz74HOzRlgV0ieUDuBLAG8V6bU+Nm24nDYix2yvoIM4vIFdC5+gzwN8XH/7zXmldn7ss4O9ZZgOSb6hNt73+7kp4eEG3zojDBQ/DFXbj12SiK7g9IxTR4Q0OMy4JqxMPqtKBIEzQ42AhEGDzXlDYRKNmagODEoUgUXEEAEB052rAuezs0bKPBVutOPGieGVrUYfBkRJWrK1rceQA5+YPsKU21Q1GFGGdfpbmByBhxJmM1+MToQPhjVKDSWJsYvR+rwarTiU+8K79fyzCzPUYEiRKXlp5vQi5GB8jOMxKlBH6wthjFGBcsVYGJOVYgKITX3p+FmmQhacooLxA2pQXKLG9Sh1+uiAiLGCwKH5wySdGxTCmppmGzMgisi8vFMFId+RMSHzwDF07But12JcPPM/Lt9ugotnc7waURjfJw7L1vkcYnPDnCPgmX+UwU4dW/DMAOPi+aB87noToX8wwDP5qCqOnrKjBxuhfeRq9zgC/YFSfe44SZNVaaEYHtlSbcpjpvICxhUKEPqAVsiuduhgoJ3x+iwxeunSyjoEQLTyfepsfXz6cHWMMkyBoqT5F2U3SY+ZqEYkBj/0MiM/XT6hyolrYPxBi/yea2G0EmgoVVpuVJ57dketXQNgqB9RkCmbG1rkH2p07Gsd6QSlD3iV+8HBm6b/Qn/QFq3GuRf4DwMABwtFz5YuNMAqUacRXoIGV9Q0Gs2SjTm6OFuumj/aDOMM/X3A+0TFxyqdTlV65dw43fAfYI7hT+6VOYePCLqwJQtj5P4ZF8Y5LfA3hLeE+KTO5XsnDjW+ZkxlTe02TzT+4LhROnZ4gutjcCDfuA+uQTCHuYan+wvM/HlqxMxTZz8WwLEmcISs6k8QYIPXabPP83hCDVucAHQIuuL80fnmxOEI8SyRsTxDZB68tKSqwegF8Czo2HyhzhP0BM9wHHZ9iJdSLQuzsa7V6B/nCUeOaxUozRKAhAe4ZXQSvEdAjbJ4SoChMXQStAXfEARlD8yewL1hlJ+m9Esr+2FhWd7U2Wl6DxqHjqBlHDh+i+MHveKUoSvQbzXmBL5xHXTFJ5WfoYVEnQ+M3xY9H8bwhJxUM+zZhgj5wtxwrik6V184YqSOJc9kBHPJHDCX6Csab2Fg0jTmoS1V9h1jvkJ56f2Th9o4mU/2b0MO4GAQdIT/uD66DKMZWXOU8ji8jEFOZ8b+BHz5QaWTWIHAHc/8YHMFjU4IiENXbG22JGzPvBlwmr6mjuLJakOkJQ42fY6Mgk/Q5dP1efBcsLWgB2RnKHA5zOwT6Blbi2eIA4iugR6gkeVK981K86cqzXxWnaQT1XZAJ+OIQj8QDDIdmyVf5SS8RgAdGwSewmbgmiQNsLf2Bej/vNICpcck4214nuqiTp1p9ElpZqrxMPYLMr5deZpx2Hc6XngpsKmQ29D7u5RPPzp1uNExNEsgvV3/xXZCFoxT/oanGXMA5gX+wn4ElTpvBOGxpwhConvhKXgfewjHnDn4ypxRFhhijFyf5TQET5BJzAM8GKlb5xWF9CDPA/sMfjvYIJByuHT3jxXcWQwDJ+T3y7b3awalO2dxixoeKOZPqEEGE2Oo/WXVDjMMECRE/Yn4oywx5DAa+IxoEBEYtt9YrsImFBmukud31lp0GAMXZoTh2YiUrCKGB8IBw45rPL29Vl7Ua5DR4Hdk+VDCRIUwgvfHWbxInSccq2QVigiov69hz6Wa8GL7QaYsMfwRGE/q9e/Xa9FumyoGBDHZEaLjZDIBAuqCMfmmeB9T5Uokii0kXtExI/inqUOA0CfLwQsjf6o6qf87t1SFe4qNGWOe+USRqTy36B/zwRgCZ5H5xbnEqMch/+f6CnOUOJ75wsDivjCWuE7gzMbaWQyeY6ycRWjhuQhB3h+ATqOdRYwvDEkikijtB9VIu29jpTyxvcaePRFMjEucFRQQzxbexYkjM8ezgg/YuP0ppTOy8xiWXItMNQqa5zhJHccPTysxo/T/1paroquQZ/SZw3vwBTREcAMnaKkaKUEEtC/OInRNFJdroxgXltfJvWr04IS/Ut5gBgDGd5HeJ4YL2XWUJ7yGwsTIzFDeJ+NHFod74x4xzMuVL3lGbCFBhJfzkRnFIWbeMHSCrB+8yTEYO0TnOe4ZvQ5zgeHDM8DAhlcDOUO26NMzRsgUlRsr9fPbVu+01vjMD3MxVA2SI/Q4+ILtXTAWYu0sgvNj6Cwiu9EV/Y1unUWVkwRQzh6Vb92JyezeqXKH7DPzDC8gEwlMwCPQOMGAs9RwZhsTni/ZkbtVvpHtWKB8gTMF/WPsIdcIMLx9XJE54BAOfIcByDWg3cqWDqMt6BBZub6+2Z5vX51F5Db6ADqB/u5VGob3oWPeY7Ti4BWkJKmeqpN/6zieUV2ySPmarCHHbdZ7xOlSNjGj+kPqHJGJ4t7/vKJM5YXqOD0f79G/XI/5gMZx/ghgvndSiVw0psDkA1UE7LMM/8EXOKToRORR4CyS3Se7gyM2Wg31f+m4ud8nlC+4Z+aQKgWcC/Qmen4gnEXmJ5bOIvov0NsHE/vjLLLHJBm5C8YUml5Abj1stFUrq6tDgcXZKr/Rp1SZYIPgRBIYw0aDj3g+9yifIKdpLAfNoqNwwuBFOuOepXz4NqUdAhzIbY6BfuFXOokT/Oc5EHijKdb+OIvIZRxegjQ4r/AvspuKGmQR38O/0BgygOfAGHAYsXUIOCKX6WpOsIXtOj44ZZjxH7/7t9Iu435Jx4JziE5Et8JT2GKM8zh4S20xgq3Q7l9W7TRdzLxwXXiVpEOks4hTSwXN+9XBxL7j/gkoUCFHhQ/8iP5mjPAMGV7mPRbOIud3Z7FviI3F6egR7POHUEN4bFCli3P1q8VbzJm7aWWZ/EGZhTQ9TH6yMjlOIszFXnQ4cQ0qECgNvX7pdrlBf0v5GaVft4edNBgQRwtHk+zbQypkblNGv3bJNrsGAu63S7bK39futGsQ4WVd44GAdYcoSQxSytkoh+VfypwQLAh9BDfjpcyIcdywvMwi9myhgNGBoCASxSb0OMsIxuv0HjEA+P3vl26TP+nxT+k9shXGBaPzVTCnW7YDowHjG0MJQ5754BhKucgILFTDA6EUCZxFhChG/9/X7LI5oXzEnoEex/FEB3HsKTty9C/IAFOWRkkYjRp47jwP6BWaoRQIg45yzDmq9PkttHWuKmKyaiirG1Zsl9++ts2eIcdDb5SuYSTjbKOsUEQ4lSg4njlOQHCNXyjdYExTzoPBQlai+xVPvccWdS4pO4IO4QuuCV9C79VqhPMdvAIfQ3eUv7YqrWLUBOtIcD4xOAJDnnmBfzjm93rMn/VvSviQKRj9ZNYt8KMOH9kGAlI/fXWznZ9SQUqhuCY8G8kXlGFfrMYWzjUNM3BKKFNnfgK+wIEl6k5pErKD7JSjfwDNEgSknJkSNBzx6/W58QyRT9AUf/M5cjAIZI3PSjPnkiABjgu88DulO47h91TUoEMsG6DXgAIwsNmSBPkZnJffQ2foCvgCupqtchMePRBAc5SkwdPQFbxH8HObGqosUYD2CBIyTnjmWh07DgOkFmylAO9jtOMkIPfvWLfL+Jdz8cLo/IeOu0oNX+YPZzpYt3vpuALLzmMwcw1+C41D6+is6PVNGP/oCwx0lq1wDPqUsTFPXBejneUXGPJkVxyxxUnDci3T1qKymyDE717bavIfGfkn1Qs84/X67AjUYWPg2CNjKdHG6XlOHZvr9JkiT6F7nu+f9ThK+1fh3CntUQFSrfT0pDqOlLQGfALtYJv8fNFmC2Zge2HzHAhwtsjGoQ+4Bjx84wrkfciZQpfh6P1J+TMYA9+z32aQ/UN/IBemK+3jDGJv3aK8wVgD2v314q2mgxraO+To4izTMzjRyBL0wCbVX9h12FTMJfNyo84LAfxoEIghAE8ZKg43OiaQO4FdyPYuOJpk6JlPR/zCncUBBsp5lhq7CKjHttZY5CUSRKSIhhGlIiJ5wrBsy1KQOSCDsLm+1SKnZDgiQc35/zy20hQe666IBn3thXXy4cdXGrOS4UDAUNpDmQLruojmhoq6DhwILozYoASOLCBCjdItIlVEYYNyTgwTjPVg7QvACCDLh2KmXJBIHdmhyDVClIJgODTpNUIRqmTb5wunGsH4cnmdRYMjgVB7tbzBau0jgYNK2RGGCYKYKDNCnlenXhIjBmMKhcKYHP2LWQXpMr8408rfyPZhTBIpBdAAz4k1U/AHkf8zRuWaQsUwQ7kR4cfpj8z4kRn87oIN8s6Hl5nhgLFA1pR1MO94aJm9hy/gRZ47mRauFR1YOBBgyG9Sno1Em46RcZLR+ZuOOWhgRDkTPAEtRgJ+JdPNHHCP3FckLLKs94XbRhYfoxqjlYwfPE6GkKxhcF8EiYhQ4zRHdvhEeRPRZi7IsrJFRcATvCjJxmgnKkz2lZJdZIqjf0BmAYdomNI4c09WInq9LnIS4w0SIhuMccj6JAw92hPcqU5UZCYDCuCYb7y4Xj72RGjNGcdihL7v0RXyg5c3WYADvqCUE6MXx8xK7Q4SqPBYXdtk9BzId/ihdfduGzNyf4fqsAD8Yoc6cOiUANA6VTM4xBjIGKKRwDB+tqzOymYJzBBcIutDoIMlG8gZgk9kmCIBT0RX03AsDgZVLMgZ+BM9Dk/g3JL9ISBJcBGnlJJGR2xBQA0HEP3wc7WBKJkMwPOi2oQsaJsqd+iGtb8z8zMtC8f3ZMyis6RkvT+qPPLhJ1ZaMI7GWjidVzy6XL6uthU8gd0S8AkvqpYibZYDAefCpgr4lww3lSXwCusjCVBEVgER0GGckZivtEsFGTpmcUWjZfsi7S70Aw4dthX2FEFbnFSqCLJUHxBsInkRCeQHn3GbkXdKACaoxsLJZnyR+oOeAyQ0OP8MfQb7apLjGHi4sxgnQEBhwEUbhgBnijKawPBF+Q/PSDImZA0XteZ9ARm790wcKl+cNVJ+fPRYufG0yfLt+aPN0cIQ6E8wUgR0X+QnkeLuFjk3d3banOH4BQIZQwDDBqDkcZSjgRKnfCoSGA04GQixH+qcXHPcuC4vjG1KmMi4OGIDFDDlZ5S+RQOFxvOPRLoqKMITOEB9BUbg5eOL5cuzR1kDG5oqYJyTyY8FGDPrv3rDGu2q1DepTMCA6A7MW6RBT2AJ+sbAxnCOvgZOIoZyEwtewiCRCa3DU1Q0fGhqSReeoBwW5xqpQYaXdTWO/gflYTzHaBBQ2dnUHgrMhfUCjgzPiHcYmsHa7t6A4BiVGu+aUGwNIX5/8iT5kf4LX8QCjBk6xjHrDSiDI0Pe3T0SuNxY37ynsRWgtBC+QCe0dCMzcMaXRDkN8ASlwSy1IEhCM6dInvjZMePk2JIc+x0OPvrWEXsYLygt8IpeX4yzBA9hWwGcfEqeCTKzlryv+oPjWEP8qRnD5aojx8i1J02U28+YZhlogvz9DQJ/lEr3FvAUyxOi+Yp5oaSc5RBkZQMgQ+CfypY2qYrSxegUbDR0VxDoBOgN+ATb7L2qJyJ5JHiREMBJxKbtb9vTcWBwZzFOAJPB7IGCjwSftL/+hvAiEkO51+v6H8f1tqHtFeogXq9CDIX/GRVqMDCLwCfnpEuGOlgYnr0704Ghb2I4lEnqLrvDbXc3Z0GpIE5pdwKUiHWQ8QyAnGJBdmFKkpytc8KC8ujXyPRQR0AXarEDjx1l1B1fYPRFP199hAZ4ozeg4y/NpW47fYoFCT4za4SVVJ5Ykm3RaXCwIsNvBrivu/vsDvyqncxLD/fJkCPPBc1iGJuc6eZ+mEcy/pFyhqkMgr0YUnRQjeaJY4qzbA0PoHwdA8HR/+jQZ88rGsFzD+mF0Gc8kUBmkY14Mxrjl5Rc/++cUtMXNMaBRy4ZW2DN1Ar1eUcahf0N7qm3lwvou7t7hFfQI5HfcK+U09p8hT7aCwQVox1PSBwdnKb0Pj03zbr/RvIEBvAcnT8qAPgt6/AdsQd0Ay3w6u7Z8nlg8fCc0P8hWgjRUW/AUqCfHztO/n7WNPna7FJrPEUTKfikICXkKMWCV/p6BcZk9l43Y4N3cK4jv2FeeE9zuWjeYr6QK2Ql+TsAvIWdSrUJgdhIHgleM/IyLBsLP/F7R/zCpVicgMgW0f/uolCsAaRLVcBMGM80rsAA4DgYLRpHqxBDcNGxjbV8rLOjfvycUflWnkGEmXI11i9StnT/5iqLNsVArvUZlHPk6/1Hg2gemUDmLDCGQoIrdBOUlna3VsBKcMPZxwAcwiL1Hc1t1hTlphVlXV6/fm2r/PSVzfIvnTdHbIBzDv12V95IyRA8EwkUIJRAI4FoQAs0mKCVOmsw2H6A93RPpVEBxgJlNqw/oYyNtRyUWUaWu8UL4Hl4oidDlHmjc2kAsiY4F2RRMGKjJQbzRTOPyH1O4SIzrvUPmpt0xxOURbHG+kcvb7JmK5ElTY7+A+u4c7qRiWQAKINDxmEAA4y7IANNlrw7XqIM78oZw83RQZSyrdE7JxRZyRqZFtYc/XXNLivHv2dDeZfS53gBdI9O6O4eoXHWKNKYJAAmccfru02HoE+iwbpDum5HInBAkAsLyutt65hovmBNFmswWc/4+Lb+bWTj6B5krIKSx+gAL29ZZkB2GBBYJqMGryAj4aNosLbuXcoTbBFBMy2aC7LdyqVji2RuYZawNdijW6ptTR/rZFkzSPa/u0D3QAM+oQokOrjHO+QH/BP5HcFE+IMKhe54K5RFfMMOAzikOJHIDxpHRfMIL9Z5slQKHUI1hCN+0b2l4YgZYCgW3SeqYKPdNp08IwFzsg6LdXLIHMruaIhBWSYKi6g+TlHAwLAqgoCGL18+YpRlD2nlfZEKNVrfk1F7ZEuVXKPMyR5631u40RSbrXHaW24MKBBOlImgkFkgPTEn1bb1iBwjEW6MfYQbv6Nsgn/ZB4t5Zd0bTQhwAgLQ1GAEi9mjmjJQsoKwYn5Z6P19nRfW8fD69ksbzFGk4yQNPfbV5c9xcECpC2UtPNvzSwtssXygvKABaJwSHwIflBJtb2gz54YmMTx7eCJYrxeAcrB3jCuSHxw11vgDhU+DDtascD2aeeD0fPPFDfL9lzdag4OtDSFaiieggDH+uXe2L6DULRLICtYPMkcNbSFHd6saLdwjUVzKqZmfwDjG8cRwmqPyIdIQQL6wJoYMJt3rkBMBT/C6asEGawABT7D+hDUy7iz2H8ik0zCjSekdQ4+sOM5PJFhKQCdOgoiUZLZ27DaZSCkesftjirONbiIBn9Bg4ouqL1hfx+8ILrK/IJ22Mei+t2Cj6Qv4AzlIFjrewBIOjFXoGyc3EjiDOH4Y/BRNM35KT6Fx/iWQRCVBdMl5sMYrEruaQlstYASz/p1mUZF88a2X1lsjLbpWsq6atVqO2IIGLTxbaJ1OnsHSFEBAjMAYtACfQDc0F2TNOPtvonPo8oytEAAbgnXfdIb++txS472Th+XYlhkELFmjSuDsuws32H6ov1i81YIq8F48AZ2KHqAKhDW7zA96IgA6lm7CfE52kbWQBFQoP0UNsuUI6wsjQQnpxOw0m6/IZYcBb7G0gYZVNPyJ5BPeE2h5VPkEZ5K1vo74hTuLAwycNxYl4zCyp+B7Jgw1w40oDUKIBfI0bIFJWZ9Iy2YWBZMZxLEZmRFq6DItN9TwhmMRgjPzMqyzKGv2aIRAswqMQzYMJirMej4EKEqS9vgIVBzPsP044Aia3tAEh7lgDzpaHZNh5T3jPlKNoitVeBMFfE2NWdZ1EskjuofgmVcYakTA/pHBMXRIo2V2pCIAZY1tFllknt85vtgMDiKSHIcBQeSd8t3bzpgqH54am82/38pgrdBzZXWmyCiVhg/ouogihwZoaHTysFxzAGk8RJc4osL8jcKnDIzMOr+FL3j27A91pj5HVBINmAhIsCYLpUZzDRpFYQTye7J2dJKEr2iaEU+gLI6GIHnK32RF2XMxoFXu872TiuV45We2RMDYR8kzn3RBhb8pMSSDhEHAcTiOBKrYKysyG0kAhTlhWyH2ybp8YtEe2cRzIIjFWra/nzlN/nn2dGt5HmmUOQ4uMLz+pQYoG8WTWSRQRgv4yGfP/mEfnjLMAhw0+KKTJ7JtkTo18MfbxxVakIXfcgzP8wNKQ0fkZ+5ZGw94jko+1vmRbo+sacXh4vds8YS+iCcoC9u2HtzvfL2/j6qMZqzwMi8abbAVD//i5KFzWaPL3pHoQ+6X7TOQ88F8EoRBxwT7ygWgS+yryk/wAGs3kUHMZ+haQ1SmJMl35o+WO86aLj86epw9J8fBAU4O897TK1RlRL+CRmFDevY5/NqcURb44NnwjAiknWMNbQgCvG7OPF3iWcOH00jAjEDicSVvyFVsCIKKBJ9xfmhURmM99reGnmggSBAZe45jSpUmqOIiEBNPMpGAHi90HvIe3co9BbRLFp0tduATEhLwCL9FflSow0jZLduxBLwF3bM8gf27o0GlGnZmTnKCXQc+COQO1+L9t+aF+OSa48Z3CXw54guu2QcYRH/pPkWEEmHHPlX/d+Z0+dXxE+S3J0y0xfJsoIphe9/GSosQA4xiSoIQTjAqdfP8nuNuOW2KdU2l6yJlMpSf0U2yUYXcEYWhjc6vPWGC/EIZ9ObTJsu1J0608opYrVnsC9jmYF1tixozGfJdFo7rWH913AS58dTJ8r0jx5pzh6N96+qde/bSe7mizhxqskrsu3XDyZNtXv6sx1yt93z00CzLzkSCbl20kcagohzrD+oY/uHk0Hz+VucK5U/0jHmM7pjnOPggE/ayGmUofTY1/p4+e9ZPoVR4JrcqjdPamzVF7B+FI4TBCE9QDokx/SE1mm84ZbL8Wn9/s/4excTWESvUKCDoQskp7fkxLogaY9j97sQQ38FD/3fWNNuLqrsy74EEjX0oHycgcooawPDF73VuuE/4gs2WS9VIWVxZbyVRAVfT7fHZHXVq3KbIN+eV2u+vU37imKuPG6dGTcjQCkAGhVI69rYjuHLl9BHmFCI3eA5/VB75tPIKa7foGrlFjYugY62jf4Dj/+iWKuOPY1SOfV+fPc/wlyoToVnWGULjNJygq/Nr+lwwZG9fs8OMYRz8b84tNX4I0csU2z+NoAl7yrJFBaBDKcHFy1T3/PXMaXLN8ePldyoH79Ln/3W9RrBONV6A3L55xQ51AusseMg+qHedPUN+o7zM62rVj8gDskpkMjbWhRqmUYXy7Rc3WDdxZADVOH86ZZLN57/PmynvnFDcJXOPEc0ec5TmjlM+Y95vO32q/Ob4iSY/kBvsw4hxRZAGeeQ4OGBZDbKnu9ePjxkrn1PbBtmPrUQAsb2Tffsy5Y+qB9Dp0Dz2w1fVgSQ7TyA90Odk7tkOYrXqEpwiKlCwN6CFG/RYbAnojPNiSxGEofKCLOT7Jw01uYhuwm6Ar9A3VIZxTDwBXQnt4vBhCzLuXyvtIkduP2OqbQ8G/ru1xragYS0iXV9DvBXaogYHj7m8Veme5ojwDrZo5K0S6P/+go1WCYezePWx4+UmtTe53rXKJzTKYr0vTuXjyiORDXUc8Qd3FgcYRICJGFO3faMaumQPMVCJcLKRMiVl7DPHVhgwLKWZgAjq3Rsq5PdLt9v2EESH+D3lQ5RYIAQxnP+jQhNmpMSO/YbYaoNGLWRYWGA8PS/Drk8LZSJjRJEor6DVejyA7OK1S7favwgqsqgYAnToS08cbIbsJ59abRkQOpghmBHilEux1xBbArCnEPN5rBpXCCSyStHbF5Cl5Rq21QgZW50j9lw6d1S+KQ6AQ0mdva9B6X9Ah7TAp0z6n+vK7bmSKaZM7lSlAYxVtpRhXyj2Jg3WZGHI3aLPne1hyJQcVZxpe0URxSTCC51/5fl1FvGEbx5WwxGHinKhWcp38BA0Bj8ReLhjbbntBYUxTWQ5HvxGsqAYLJ9/do0Zo8VpiTZm7hMHgkDIX9To+ZXOzSJ1uAMFTqkU65NvX73TjCiy9WeMzLVMEfcK/SMLAhCgIouJw8j6Zsq0aaUOT/AcpuVmWLCLfbkoK2Lucdgd/Qem95/rKyzASAUFWRCeIZliysdw8Ngv8KOPrzIHHr7gxZYRX3purZVFsiYLfkAmHq38gZHHOu1bVpXtKeFH37BcgfI75OZ5+sxZo8VemgTl+B46G55Gtr/ruvCBAOv42X+XvUvZXxgddqbODQ3LqBKgpPxXr22xcmoy84CAC91Ov6tGLaVwzCfGMpknHGuWZ7DnXiRYg0Z2hhJsNjSn9JXSX/gP/UuJIryAvmb/yqDjpuPAcURBpsme7l5k8k5SOUjwCnl+t/IJpfLIepaesOUY+43yrLAVblIZyRYxZAkB8o5yetar40yRBYN+4C02qseWoGzyen3uyEIC0jxfgpXwQGAvUOmBrmEPaAIwyFBoERkdD2AbJO6b9bR0PaUnALKAyi3272YLKUqrb1eZgHxHf6AXkDnQNPoHXYgNiSNOJU+oOu6NDswAumf5wg/VFntBdTVLIIJGNziPbOuzWL/HprpNdRJ2qiN+Mej13rbSPMxB+c28OxYaI/QXUEQINZQJTh9CCsclAGsjUPiU2wUb4yP02CuI0hciNJHMiCIfr0pwbmGGrUvk/HzNMUTH2P8mcp9BMjGUXrCfYLCnDcKAqCrGIiV7k3JSrakHhjrMy5gobcKBekYNjTdrbMAm4AhGMnAY4ZGgRp4mOyj1O9R5RaFGgsgdrZYxgMkoBmC9GqVANCShVAGDHYXNlhnr65vlfnWIu2utTrSLa1ImhPDGAaFMFQOBEgpK7n63ZJsZS4D5pOSUZieU8CLcqOGndh+Bv0LnFAMMQyQAmWBK+Jhv1qn0N3B8F142zyKWscAXnl1rG/D2JyjVYW0Q2zNQBvTApqo9fAhNs4k2a3kp74JuedTQ+KrqRqXxhi57jPKs6fCLcQu9YEDz7KE76BKnKOAjsgZcl5IjMvs0v4C2WKvB+jv20uJclGrCq+wnxbGUZbLGD/4iyxfJl9GgpJlMA/fJPp/wJZmfAJdPKLZuo+yJFb3PKqVC0CP4qypv+BCQ7aScB6XLeaEHjFbuE1rFSCGTFJ3pI0M4We8X3kBZMzcodc6LDORzHAYcAqoXAuBcIAsoyQ1oj6Y5VDpQEg+/BjNAGTwl9YwLowt+jrzf/sD1J02Sj0+LTXk4Gen5dy4Mv+s/IKEps0dWvbiz3jJiAXBkoA2MtqB8iyYdNWr8rdbnRjAhGpzvvNH5+nzSJFflLLINA5nSaxwisvjBuiFoC+MXfcH1Ac+7TmmF4AlylBJkPoOfyM4gp989cajk6O/vUT6BznsCY2H5A0E/6Ik9PNkAPwByDsO/VPUaRvkryueBTIDO0aPIhBXqBBMEjQQldPA+QVPK2NEX3BfO4sv67Mh4BDI/APdIUJD5xElgbtCJ3BfjI6CCEc298jkgM4PxC+8im5LDsomSXeg94CEcEz2dzSXXQKZxXkof+xM4vU9dPCf8rv/BfoMEog420N8sDWG+mceegOyj0oS9QoOgF3IbJx7atK6begL4hIwgTh4yPXieAbClcKB45pT6A54h8pHAQiRdI9vRT/wb0FogF7HboCsCj+iTIIsJD6JTGAM2IHJ6X0A3UkKNffI7HDy9xwDouk9NZynOIJO1kXYTwEGGDqB5aBe9CaBpAih0bR2arvamjh3aRV+gCx7bWt3FPgPIHGgYO4lj0JXoX+ab+wSsacYeAswHjjTl8vSdQMcSrALMOyX13H/kfpDoGvQHPIiDGv18DgbY7obMsaP3cGcxjFg4iw7HgeJwdBYdjgPF4egsOhwHgsPFWXQ4DjbcWew7vAzV4XA4HA6Hw+FwOBxd4M6iw+FwOBwOh8PhcDi6wJ1Fh8PhcDgcDofD4XB0gTuLDofD4XA4HA6Hw+HoAncWHQ6Hw+FwOBwOh8PRBe4sOhwOh8PhcDgcDoejC9xZdDgcDofD4XA4HA5HF7iz6HA4HA6Hw+FwOByOLnBn0eFwOBwOh8PhcDgcXeDO4lsUQwYPkuHpyXLM0Gx9ZcnozJTwNyKzCzPlpGE5MiknLfzJvpGTPETm6DGnjsi1Y7KSEsLfHFykDhksJ+u4ji/JtrE7HP2BkrQkOXV4rhyrvJGZGKLlwtQko7vTlMazk4bYZ2+GYj1mrvIFx8EXg8KfH2yMzUqVE5UvjirOkkH9dRHHWxqQ1ayCDDl5eI5MDusFPpuof0Pf85TOi5VvegP453TlI44rUh7pD6QNSZAjwuOdnpcuuaqjHI6DjcLURJlXlCmnqL4Yp3IYustU+wd5j67ALsI+6g3gJeQ4x/QXClISjY+5DnyRlOAugKN3cEp5iwKhdt7ofLnp1Mny51MmyzsnFIW/Efnp0WPlzrOny6dmDA9/sm9MU6Hz02PGyYPnz5LPzBwhE7J752R2BwyQRHVkM9RIT9dXQoT1OywtWe4+Z4b8/cxpck5pfvhTh+Pg4syRefLA+TPlltOmyPiwYYxhe9sZ0+SRC48weu8NTlJD9RfHjZe/njHVeClB6fpAkKKKHb7g30i8d9JQ+b+zpskfTp4kQ9xbdPQDBitdff/IsfKvc2bKp1XGB599avpwuV3p+xfHT7AAY28wJTdNHlY+ul356QTlqwMB+oEgYpY6oJG6YmRGsnz/qDE63hnytTmlMjM/I/yNw3HwcHxJjvxaaf/+82bK+ycPleFKd9g/V84YoTpklvxM7SKcst4AXrpL7S6OOVCgB7DxeGFPBZhfnKV8PEbuPGu6/O/cUsnzIIqjl3Bn0RFXyFSlf/LwXPn5sePkeyrUxma9kfF0ON7KeM/EYrlGnc8PTSkJf+JwvLUxISdVjewRFiwZlenVJg4HQZOp6qB+XZ3Bb88fbVlPh+NA4c6iowt++PIm+fDjK+WmFTvCn8QOparwiVCT3aGsI2XIGyWtO5vb5IOPrZCPP7laHttaHf7U4eh/vLSzTj715Cp550PLZFVNU/jT2OLsUXlyYkmODI0qwb5z3S756OOr5CvPrZPO118Pf+pw9C92K63dvLJM+WK1fOelDfL8jtrwN7EDWZy3jSmQ+UVZlkUJsL2pVX76ymb5wGMr5bdLtsry6sbwNw5H/2JDXbPcpHzx7keWyfcXbpSV1bHVF0kJg+Tc0nzTF1Nz06xCK8Ci8nr52aub5SNPKF+8tlVqWjvC3zgc+4Y7i44ueHJ7jfxrQ4W8WlEf/iR2oJxoeHqSrX9B6EUW1TW0d8o9Oq5/b6yQ9SqQHY5YYVtjq/xnU6XcoY5ZZUt7+NPYYkRGiuSlJNp640gsq2qUe5UnHt5SpQZ8+EOHo58Bqb1a0SD/Vr54fFu1bGloDX0RQ7D+sTQzRTISB0skW9S3dcozZbVy9/pyeXFnnZQ3DwzPOt56qFYH7BV1yu5cV262VEWM9QWZxTFZKbb2PjKAAsqa2owvsO9eUL5o6dwd/sbh2DcSrlKE/35Lo06Vyx+Xb5fGjs7wJwMD1iSNz06T6fnpkquGYYdaf00dezM0i5RZDM0i6rQhg6VZv0dx54c/p+EF5xiXnSqjM1Nt/UZxWrK0qWBo0/NhUCYnDLZr0OCGYxdXNsizKkTA7IIMOw+/iTSMMVTH6Lm5Lmu5aIozVAUS15pTmCGj9D1CEgMCoYTuZn3VaBVc4/Q343U8vBjfKDV8h6lTmKrCrFGdQKLUfE6TjiP1xXXq9fNdquSZg45wxuQ4HS/306Fv+T4ADQy4DmML5oBr0ECBmv3atr0jaDij84tCi89pYoLBwbHMGX/T0ITxI0zjyQDn2X902rCYLUx/SB0QlMpAA76YlJsmk3PTzUAsb2kzug2AgiTQQFMLnh8OVRA1HaX0MjYrzbIQlK1BG9AuTZI4hufbEsFjnOOC0QVKM53qHJbLDqVlGhnMLsjU49OkqqVDWsNKlusUpCgP6HmNL4znUoy+oCXOxXhX1jTZXAa0xGdkzuGjCfo7fjsGXtVj4Snup333bnvO3M8Reu1LxhYavVYoT2CA8DdGMU7kjLwMGar8REYlAOfI1t+wxpJxcz34imtwPy3KRMiDyHmckZ9hY8nR8cH/0/S8zBufjdTrcE3uvT3quIHG+fq8aDQRC2xvbJM/qa6IByC7jlaZOEafKc+FgFo0SpQuaJqBPKxqbbdnDj9BN8hKni20MVb/RV+UKF9gZAb6ArA+8V0Tio22X1Ndcb86iMhHjp+sfImc5aeB/kTmDlWdQzmc8VxAP0pXrOviXOjchzZXyYpw5oXzQfsYutAcvMTY4NcR6SmqD0PrEgO5P0VlwRkjc+XIcFZxkeqd9ET9jYrGts7X7do04+Fe0SGRhjE6BH4NrsM1hul981t0UXPEb5G5XAvZU6fXRm7wd6BnRjBfXEOPiyfjG7nxwRiWrP9THfMllfGRweW5TNVnxLNqUznaovQQKa+gI+wfniM8hI2FTEMfwEvQrMnnMG1Au9A49gzOYADogkZoJUrrz+2oVd5olHalAeTsLJWlOUo70ERAF1wXPcb5ObfJY6XF7OREW34DTe1UffOXVW9UdSGHi5QvWOvLWLCl4At4lfvD7kN/cQ30CusSqcyCV5EJ1Wr7cO+7mtvs+4l6PNehEQ86LqhGYe0vx/Ad+iy4BnzBfbfq/DBHAHkAr8JjRapL4D/ojWOZt9Kws0pWs06vHz4sbkCDHxoyOnqPQa8rwn+/pUFUdN4dC42hBhIzVICx0JmmFS/tqpdfL95i2bRIUHbzCf3N0SoUECq/W7JVals75ZQROfKhKcNkvDJrgTIwQgbmrldmJTPy01c3W/kmhiZC8V0TiuQLs0aacrxFz0PZDnj4glnWEfVva3bKZ55eY58lqfI/a1S+rZuiTBTHEWN2S32rlYem6rUwSHC4b1xRJi+r08gxGAmM9Xhz8kLGJmhSowJHkGzIr/QeN9S1yDfnlZoRMVGPQRgxLoQMWRPOu7amWRZeNs8MkqsWbJQb9DMMDO4Txr9sXKF1sUS4QdQ4oRvqm+Xu9RXyS70G7wNif9f4Irnh1MnmIK+pbbZyDZyBtMTBZlisUgPmZp2Tu9RZYO7iBRg4zAFGTSzwhWfX2twNNGgS8M15o804JDL6roeXWZAkAEoJo/G3J0wwerhxZZnS8yalywT5zMzhqjzzTUlj+AFoFz54YUed0fl9Gyv3KM33K+/98eRJslllwjv0Oq8qLV+sjtovj59gjufx/3zFDAMlUclXJU/JzxXKF/AMxjBKe1N9ixnVNNagOzA8zFxCWziAxw3Nkg9PHaYOTpYMU6XKmKF3+AJn5K86JrIi4IqJQ+ULR4y03wB+hyP8VFmNfO359XK58swnpg8zp3a+yjB4Hv4hgELg5Ts6b6z9ZT0wwKHYqOOjROo5nUsCQgFf3HHWdHNwCR4t2FUnH5xcYgYJhgQBl5dVJv1q8VZ5Vu8/OgAzkLj+pEny8WnDwu/6Fwt1DubfuTD8bmDBs6KxGPQPfVEWGhh0AOMUh+EnR481w/HM+xZbGTWy7rLxhXKOyvTAqKMpBob1dqUj9MRNKsfRQdAb56G5GLL/ttU75FNPrbbPaOB00dgCpfdW00P/t3aXXReauVTl8UeUxjFAqRLBOVyws05lcovpBHTuF55ZI3cpnauqMPp+t9IygRruCyOYz5DxlerkvqbOILrg98u2S6fe4x9Pmaz0OdRond/AWztUF/1+6TbL6lx93Dgr24a3ce7J8gBkJw2nLhtXZMY37zm+rKnVfsM9PLyl2pxlZvL80fny1dmlxkvvfGipGZoXjSlUZzvZjqtS/nlAnV549r86b/FSCk7zoKcunhN+1/+44tHlcvvqneF3A4tvqcz7gNIGNP89lXP3b6oypyUA9P6X06bYsyRg8WPVFWuULwi8IP+PC3fqDWgDWQe9Yq9ctWCDyXgeM3rhSyqb5yg//XzRZrlV7x/nE7n5MZVHTyk98Xt0FvwCTf/kmLHWMA0Hk89qlLYXljdYUAdHlaDH6fcusnHq1zJWHbazRuXJ/ygfw6vYUImDse1Uh6kNtaauWX788ia7FvqEpoU4bvAOY4R/cXCPumuhOcef0HEdXZwtj2ytki+qzEBvYKvxHdc4Wx1NAjocDw9g3z2oc4RsWaxjI4CE84j9+NmZI/Xc7Za9x/4j0YBtCS8iR54O3z86LZ6WSHxjbqn84Kix4XeO3sDLUOMMq9Uhul6VIcYgERqis9HAGUIY4NhSd46jdaE6kP87Z7Rly5o7O62EFOGxorpRElVREwH7zIwR1qFxf0Dk/qOq+BFaCBYM6EUq4FRmmGKfqa/obo8FKmyvPna8vEOVMoJ3U0OLjYkXwovI1HlqaH/piFGSosYoRgznxUhH2GxVAfOSGqyv6HUQNt2BzAnGEAYxRgaSHcGFIY2TNzw9xYzy69SYJPobiSQVuLbmRY9jTlfo9V/aWS9N7bstQobgxBB3DDxWq0O/vKrRHEQMPDrjRmZXs/TZEkRBGZPdfkGdGXjnp8eOkw/p8ycqi1EX0B/GbUbiEGtvTsMYZZE+A0cR4/lb80otQ9+kThhBEhwtxnG2GuJkDaNxiRrXdKLDUCfIwu8ZE/82Ku0RccbIRvkS+V2q9833GDvN6kzC70+r8QGtdpdJAkSrMZauVeeZVukVLR3m5CzQF3ID5/vXx49Xh6HIeCgS8OKxahjD7zgPzCdjwDfHmKL5lDcTiQ/w/J/bUWcOIgGzE9Q5igRZ5SPCnUD5HYE2giofnlpiBm2h0in0RNDkRZW1GMSlGSnyTqWL00fkGR3tDzg3+iakp0LG5FY993QdCwHHaMAvyFqMT7Zywsg3HaZyfJHyBS4jBjy/IWMDja5WeU0JNnOgt68Gd70F/7ifnkDG87oTJ1pTHCphqICBtxgf5zhXeZKgVHd8gTP9vSPHGt/CiwSMlql+xfG4WHmaOUNvOAYePBvsBqo+Ligt6BJcRXNAm9AO61k317dIqeoVmurRJR5HCfqC/qAN9A5y+QL9jo7vueHAW19AsJLOqdg8hTquLWoPYd/sUP6AtpHJQTA9QG5SolwyrlC+OmeUOZIEOKFz7m+t6kSC9iQN0GHoolq1qxgvPEDgsaKlzSq9CO5FVs9E40y16+ho/z51lPNTE60SBr5YqnMTSiwUy1XzQ3MTjel5GXLxmELTsev1uuhA+ArHFpvsc7NGKl901YOOQwvuLMYZUJKsryAzka4KkYwAUdoAZOfILpHB2KwGb7kKD4xiPiOTwGLqb764Qb6lLyI6X31+nWUfalS5UTIwMuJcfQH7Vc0qSLeM25f1nN94cb1886X18o0XNsjf1+yyyHCkvZ2phniozCfVSjz+vnaXjmOdjYnXlU+tlv9srLAxUz6GIn56e63cv7nKDFOMHwT4H9RxpoEH89EdUAKU51EyQSbnN+o8M7arXtpgkXayM4yLzCYRQwRYAK6Jw4ogvWbRFr2X9fId5uyFdeqcNNm8j8v2bqzxACKcBBswODEsLxyTr87aG4qVQACRdMpCcbooByL6icOVpbRIpoEoakB/n392jdFGa+frqsTTLWve173YKO1kyxmMTjISNA74X6Whbyr9feLJVRaoCcqnI0E5ICWnZDI++8wa+z1jgk+/u3CjZV5wRDECoEGis0THuXfKqZ7fWWtZU8q+cIC7A6VR56mRlKvnQekTOec639LXDxZusjVmGAHwzlkj88JHhYADS5k3juVXn1sn31Je+tJzay07Q5UCvEZDEXjHMbAgqHifylGcF0ouo40yZCsvgg7/3lgpda2dVgKHE4nx+CuVl8g8Xsg/mptBG4FzBv/0BegBgjanqL7AoSPT9umnV5s++orKf6pOyroJ/KEvoNnhGUlKr7Xyk1c2h3SY0h467C+rykwHkKUkY0k53T0byi1jT8UIWYvrl26TX6gcf1oN/O5AwJIACoEdxnmfZfvXGO+hM3712hZ5RR1UgrQfUF0RLQ/I8rA28h+qy7iXb+vY0GmUl1O2irHPPTgGHmTnlqgeIAgIrSdHRAOxAa4I2wLbG1vNzsDhP214rtHXFn1PxdbXlR+gP2jjBy9vtEAEJdonD8u15TN9ATprdn6G8QVBid8prX5e9RE0xPn/unqHZfjINEZiptpcBOP5/ImwHvi28gXH0VTq18q/rcrHLFNgOdCWxharwsIBJugNz9y1fpfpi8jy2UigYwggwevw0s9Vj3Fu+IJ/qUBBz/D9cUNzzAaNRLryBJl/qr3gWe7np69usowtzi9Li4KKHsehC3cW4xAo8adUMDV0dFppQOQeUaT5ieAQIcbJwamiHIFOdCjL6/TFehKMV0ofUNYPbq60UjPWN/ZVyBGJxeFDEVL6gBDG8KZM6YltNfLvTRXy6NYqMwAodwjA30TBbl4ZKpOl1I+xMCZeNAtZok4hSh6BgowkE0gnsYpmPZe8LvXtHXY9Mo6RJSQBUOYIsJn56RZFYzzBonKuQckIJUXMU1EaDkaBrUUMwCoGnHPmB8MDI4lzYIQTmePeMWIc8QEyXERUU1Tx0w48iMJiOBKYYP0H6z3W1zYbvUMzBBrohnirKmOMuoD+UGTQVk1buwUcMKAxbvsCykdRsqz9o2HAPyNoCCMW42JnU1dnjvvAaP7dkm1yr/6O3zMmjoWvMF6gf4JA2ckJslX5gnOxVouM+zblNdaREvkO1sJEg7Jq5AQ8RcCFplDwBS/kw++Xbrdg0wzl67lRDgEON1FrOvo9qHOGs/nolmqTMZQkkdElyh0dBXfEHtAEz5SMYH7KEKuQiATZENaGb25osd+hUyjnp7we54hA36NKczxjvocGyXYgTwkGFKf2zcijIJTAItcly4DcD/TRI0pLyNb7Vd5GA1n/jDp51ypPwBuUm3IMfAFPIMMxfNFf8DqZnzV6H7zQgfxHJpKxMxfdgfmhbJD1xGRScfqQA1yDazEn6CgMZniCF0Z+AHTasuom/U2V6RbjC/09GVvKAekxQDdvx8AD+U8TPMqSoX8qiAiOAZbonFeaZw4icpXMG2tcCUYSELtOZSN2w2PKCwFtsJSFQALVSKwfjG4y9mYgi3nU0GzLBJINf0h5ANqDhuAL7CQqZ7BHIlHd0mH0TxD8z8oXD26qsvFwHPoM/UZ2FFnMGkTWsFMqyv1jF1L6TbXavprZsHwCO4pb4nforkf13AHvwbPoXfut2p9HFe+tL7g+646Nf8L6794NlWYXck5b69lHu9MRf3BnMQ6Boibqua2hzZRuZHSXKDHRLwzKB1QJk3EhIsW6D7IHrJtAwXEcETWO5dVX4RYApYwwwcmiNh0BjBEeuIWUZ2C8ktFUebsHOLNE9hgTa97WqfHJuIl+4/yy/iNk2OzfuABlrvN0bDh0G9XwJbMYnYHEeMAAIFLOnOAwB2C8la0dNvagWx4GAdH6Rr0vIoAHupG64+CB5gkoM7KB0DZGH/RJZJQSHNaK4ABuUQcJEKwg00BkFEXM76AB6I9ybbrukl3eX5DNhNfK1SCx0puojMnzOtbuMuIYmPAFhgnHwxc4dvAZgQ/uiXVYBwIMV9YZwnc4spFRZdbfYIyvU17GQcaAiQZ8jjETEf+R1rDxAbh3DCfHwALjcmN9s6yqCTUWgZbIdAMar5DF5hFSEcLv0Bc4Xshk1p4SLCHqT5MaDEGCgsjTA6E/rgl9cE2CDJFYW9tkxjHjjgwuIn8xmMmYYAxzfRrdwKs4bTS4gS8OBBisOA3g8e3VlqGPBE4mMoZlEASQuC7NOwIw3KWqY5ArAfhsZ1Or6TuSV5Gl8Y6BBc9xcUWjrdVmuQFOI3YQ1RY0v8F2sVJi1Q00fsKeIrPOmtwy1SE0zLMybuULKrwISu4v0D3wFsFxdBil2QEIAKI/6J0QyNcAVMngJF6teoxADqXRlDrDF4yNgM6B2igEFrk/eJDADhUvQWMrAjEEQp5UB5BALM3RaIIWCaoWCEqiVwKeZqkDwSJA0N3NqEMfLtniEESALJLa0GJrSiblpBrDIegwdhF+ts5EhQ7MDFhHiIJDeLx9XKFcOWO4raX65XHjrTEHTW/2R9mitIlacX2cqEhFGQDnkSxH9AJm7A26ZBHRoxSBNYBfnj1Kfnj0GGsgwvqTvpb+9YRNagh1NzaiXmQqiVxHDc9xCII1GWTaMIpn5WPMJVmjiZPVWeTxEnFdXxtSUtAfpag4kZTp0ADp0zNGyI+OHmtrWFmfQcR3f4FxCH9QghPwYSQwRGgkEw14KV35gjKoc0blyQcml9gGyvDqT44Zp0Zqxl5Bjf0FGRIi6z0Bft7XOhbHoQHk2lJ1cggIEBi8fHyRGWcXjS00g5LuvcuqGsK/DjW9IZBAsOWY4mzTF8jlq5X+fnz0OHnPAcplKlBwSjHAeUWCJQkYyhimkfoCWxK+QIfhuFIa/eEpw6wZyG9OmCCfmzXCSp8PFFyRbM2upnbLwkQDfbEjoqOw49AF1RFkDqF31o8TxEAXIPMJqOBM0t02ALKcbCNdgo9XfUKzwO/MHyM/P3a8/PmUydZ8aX8BbVMKS2Mm49VubBXsl+imYfAFdhtZ8ck56bZsgKaEPzxqrPzuhInyp5MnS3FEtdSBgGwk2cDIxnGApRRUgfVUxup4a8CdxTgG2RAECOV1RMbouIXA4zMWIEfibWMK5e9nTpMnL5pthuf7JpVYqR4NClo6Os2hI4LVHyAKhdCNdsaIWF+jBshjbztC/nzqZPnk9OFWBsR9JKtB0aAOHE1BHI7egvV8/7HOpWLZRNbP0W2ULDXZMLLLbKsBoD+6iN573gy55bQpyhej5YpJxRa4yEtOtHVGkcbCwQYRWTIO0Thdefm3J06Qhy6YZc4hgR2aZhAIwmApb+4/XnUcfoBS6GZNdoLMA400MDPprI3TR1dbujQGwIH8qjqH9583U/50yiTrDHhp2LEk07xL+ae7tbYHC2QfQpnF8AcKxo1Bjw7jRRMlutvSsIdtn/g9jpzD0VuQ3UIfsLSFrCJlmvRtOEPlL8E9SvODyg8cMjLZfzh5ktx77gy5/sSJ8kmlv7NG5toyHNZw9zf9UTbKKxJkQQlq3nPODLn7nOlCAoAupEcWZxpv07MiOkjvcPQH3FmMY5A5JDpGCQPdpugiRxZldXWTdTYMQBSWRf+USyDQqB3/2gvr5JNPrZbPPrPWSivIMOyPAYBiDxxNMoQY2dGgE2XI0A1/oCDzg5A7e1SeHfNqeYMtxv74k6vkyqfWyBeeW2vtmJvUCDgYoEypu7ERKaTMjn/VDncc4tjWEGpvTwSWDMS5Sl+0x8fIpeyZ6Ce0SskQ2wOQJZmUnWZGAQv/P/HkaqPBzz6z2soziabuL3BY4Q8UenflrPNVoUd3kySqzTYebPFCGSdrn2gIQEMcmj798GUa3DSbcXygIKtflNJz1Jk5OtDSPkd8ALqnFAx6ZE3flHCpGhKfz8kiByBYQdCOroes3fvF4i3y2afXGG98b8FGKx3tLuvWW2DwUo5JRUp361rJPJLVjFwaQZb90zOG27KCSnVWWTP2deWLDz2+Uj73zBq5aeUOWy5woOCKOckJtm6NLGY00BOR69odhy7QA2S2aZiHfGa7igtHF5hegB8o6wyWDiCnPzdzpFWogMe21ViTpY88ga5YI19Re6W7tba9Bc4p6yIJBhLMie60C9AjwbrKAGzbgnNIwoCyzptW7JAvPbfOeJUGft9bsOGAdFgkcD4JzlCNEwkysywf6m7MjrcO3FKIY2D80lERpUo7cTppwbjU2PMKwLYVU/PSLLP3vCp/Onn9bfVOcxqfUIHIthMYhRzbV5A1pGwCgYQwwUBn64LgXAgWsjpEpSOVP5t+0xmOJiAY6v/ZVGHjwgi4e0O5rf2g3CFaMPUFlHKgCMjeYIQzB9ENHmizjpKg8QLrPHta5O04NMCzhp5YDwU9sk8W61/p0gltEWkFGKmhDcHTrPSNdasB/cEXj6sxAE0fCP2xjQVGB6Xi8Ge0Y3jKsFwr/44EvAMPYRSwRoVmHqwzZvN/mj7RnIAS1f3h1UgQ4IF3iZZTlRBphGAk46wS0MGIiS59chyaWFHVZMsT2Ej7PZOGWhMisirRThaBPZ4967IIoNy2aqd1q6Y78As7a81higz89RV0mKS8GQOT7TcigZymazE0GEnjrJOk+ypO5uNba6y50l9Vh8GvZIBYF8xejQeC0PrOFjPY0U2RvQAADitjho/pG4COrWxx3jiUwTo7msDUtXfIUUXZVm1FkISmeegRnjPAUWNbMapRCNLfprqCtYvQH0FFSqcPZH0742CpDjYSe+zSDTvQPdAjegFZHR1cmZ6fLmx7xvILxkOjNppG0XSGLTLgl0EHqCtY1856XWy2c0rzrayVslnAeAm2nzAs25pdITO4D8dbD+4sxjFoXINBSWMZlCwvSuzW1DZZiUUAWzulzhDlCOyLhuwgEoUyZO8t9oBCAEQ6c70Fa09wyHipXWmRX7KYLP4nk3m8ChGcMTYrR+gF4G+7nv7PmBgbxj17vrH2hMg252J7kEiwzorSQO4Hg4IuldYwQQ3oaFCOS4dGHAe6RlKWyB5Gtu+jOolch7GepgYLc8jGyd2ta3QcWsBhpI04kWOitKw/YR3jvRsr92wlEawLJNNCRp3mLBjBBA+gXRyo6AYWfQVBGBoocd53jC8y2qNJjfGF8sSJyhvDohxI2ozDF6xdIQND8AJDgeYiHIMTR2ltdxm/dv0tLMaYCc5g1PbUUANjiGYizA3l30TMGRcvHGw2Jcc4oXSXTnaOQx+Uob68q94MzvdNKrYAAduf0O0xEshm/iNQQFABI5EABp19ydKfpHTbXdatN6ArKWt1oSt48/2Th9oWMNAdzdkuGlNg283Al/y3B0rXOI+v6+cEd+ARdBh8cZw6dsj27vYs3aPz9D/WdNG4pCeeJrh4/6YK0wGcj7Wa3DO6gvFhKBOQpUIl6CAZqWcdhx541mwZQwCjVOUdW8sgd6lOiV6bB/1hwTQoT7QrXZFhhjaOUfqlSgr5DH3uD6gWgS8IzLF+/tQROXKsng976Ejli3eMLzTaTUvYm++wgxgTvEr1AA4mW7tgO7F04RMq21mHGQlGiBOM3kM30aEXfdGTrkBGsCclzjCBnNNH5prtxL1zHcp2g4wjW1LR/dfx1oM7i3EONm4lWkxTG15EgaI3qGeDcVraU9pzxcRi+fkx4+Vnx46zxgA0zWCDb5QzBipKOIga9QUPbK60TpPjVdiy3oW1iL86foJcf+IkW1sSLUKDpjcIZAQQG++zQf/Pjhknfzp5klw1f7QpaIxlRoNxjBFD5JcIOUMcn5Umn505wsowiIR3B8puyVTiWCPQqOn/xXETrDHCDadMtr0VEaZkR3+/dJvtVec4tEGJ3H1qAGAIUEoJrVCiE0mDdGijhJsyJJyiT6lS/Y3SK3xx7YkTrcFSsOcV9IeiDKnl3oNOdWx/QTdE1kF+dU6pbbpMQ6l/nj3dFG505nJTXasZLslDBlsg53c6FtZnGS+dNEm+rXwRjIIgUGT2hfvlbDh+/A4HNXKvyUiwVx3BEfiejZR/pPzwC+VZ+JbNlynH4syUwT6+de+OlY5DEwQIWKPFGnWWLmAw8j46EwAdEXChHO+2M6YaTSCbWb8ILeJkIovhq75mLeCnv61h24Eayy6eorT6l9OnWvOcPyh9f015BCMUvRBpdwdZP7bGuHLmcPm9/hZehZf+duY0OVcduWBrm8CoB8h/mtUw1q+oXvqmyn+yR92BjcLZmiOUvR8s75k41NbSoyvQGd+eN9qOxUEkG+UZ98MD0OS62haT8ey5C72w7j1yDWJLZ6dsrGu2dexscI8c/7HSxU+VBqFf9uccnpFsQUdozXgjfGxvQEMngjkEOTkWJ49rwHfoo2/OHW16ivNHAlqEXwma//6kicYTjOu68N84mgA5H+JXMTsPO7G2tVMdyxS1CYfKd1Rf5PWgK9hWZ+GuOgsaYkP+6KixJhPgC/gWvUGigsQFwVGqFRxvPbizGOd4uqzWSg4QZrzYG4qIZySsxHPlDtunChE2uzCUpSAyu1Udy1v0O9r1YzgTST0zahPu3oC9c9iX68/Ly+w8OHoYwzieL6oAoVSDLGSAdWq8/0l/y3gpcaAclVI9jGraqrNe8Z96zDJ1DDFOPji5REakJ6mT2anHNlsJUF7KEJlXGNosdlhG9+tI+P1NK8psrQ2bU2N/sA8QEXK2RrB95pZvkw8/sbLbZiOOQw/NygevVdTLyppGKwMlmELLb5RkADKMbM79w4Wb9LeNlm0hkwj90TGY/dloCgLdkoEk0JCd1LdsCmWo7Jl4+SPLzTFDWbP1BdmQ+vbdVuL3UhSvrq5t2lMOy6bJwUberDPDYKZclq0+yJRSLsjnAbgW9Mz4GS/ltz1lgLaqAXDLyjL59NNr9P4bpCA50e6fLUPIvCyrbpTPPbvW2rJjLDgOD7A2neg/nLBY/6X8LRo3K11cu3SbBRip2ggyf1RvsLfaOx9eZg4mmZWh+7l+j711v6t0/FpVg+QoX6F32IoAQ5MycMrFI5s4sTfqR59YZdmXRLV4yYQwLrKJVIX8/NUt8sdl2y2AEuyzaMfp+dgPjwAS288cXZyt1+magQxAhuZTT622+2dZAqWn6Ap0BvzLurTvqi75k/IFXWQdhz7IJAbNbKrbaA7YaEGUSMeMTDJrE5G/VDZRwn2CyuW5yhfQKeWfyGzsjVHpyUo3yXuCF70FzhZrH9kPm2oo5Dj6iOoPsv8sIYreI/QGtaH4/RrlR/TUUcpHZDrpAs49QKvYSvAuASKy4tD4I1uqjfcI9sBL6Jh9LbkgaEifi1tVN1F6TTk2fIF+grf+vnanra1HXzremhj0+v7m1Q8zwKTz7lgYl2UnCAEcLYDz2J0BwG+ITCFEcOBU76kwDO0ZyNosWqUjnMwZU6HFXjoIqcAYRYkH5WhsqkwNP8IVAzUABrft3aXHka3jGuxHWKPKnH8pj0B5c64qvS4KnfKmYSpYQ2tURBDPRLy5B2wFzkl2CAeRcRHt4x4wuqmVR7xhIJMlYgNqHF3KK5apIcQxATiGcgs2Mg9KWxGaHIMxHL1uh3vAGGEsCOnoReJ8x/pHMpYYMPECntfCy+btd5lYX8EievZki0egaHlGZABWqMLkWUUKM/gAmmdfKkqgiSxDf2Q82EqFdSgoUPgGBQmtcy7ez1OlTKdenj3lPzSWIhPDlhYYFER7AQZmSkKC0HkSGoTPGAPHQp+Uf0PfOH9sGo60ZSxEasmWwyPwEesLoUWuxX2gpDkP/BTQLs+e4+AZHONdOoZFes5SNfhxLMnSo/ThK0AVgZV+52dYJ0AaiwD4Aj7D2IDuI4121o0xZjLwRJEjwVxSKkXZIlksMkGRjVMGEmRlqXCIBWguNv/OheF38QUyAxh6PCsyadBOtE6D5ihhhp6CEjZkKvRDB1+WHGBcQutk7KETaJTtBIpUJvPcyZLwGWXXQRaTAExk4KEknbGkG73Ci2Q/cMDgMcYHyM5Da2QyuR6BFpxUtoKCV6FVeIkgS6ryGQ4hRj5b5KDLGD+8OVnpkmMIuGCUU5EzS/mV8W5XXUdmKXIekAnwEs3Zgkobyl/hPbasitSxyBiuy56ozCf3T+UM4Eh0CedjawTsCDKX8QAqep66eE74Xf/jikeXy+0RXXfjBTxe+GKa0gh6M1g+EA30w1wLpoXkOM+WMmfogsACdEJ2Ef6B/pDV0DH8hr2EE7pR+QW5y9IClhOw1hZ+CvZxJivO5/AfeoFzIn/ZfgmZnqGfURnAmvoABHRC69xD63zhC+icsmr0CkmBHD0O3qOCiu9wZGfkZYTGq9dAv6Ab0ANUbLEVB8EiGg8GfRyYG8YGP+N8DtEJoIkbQVFsMGRBkG1nHKPU3oLumTdsRe4zEswN5d7MHx2Z4a14AR2gf3DU2PA7R2/gzmIY8ewsOhwB3Fl0OLrCnUWHY2+4s+hwdA93FvuOvuXRHQ6Hw+FwOBwOh8PxloA7iw6Hw+FwOBwOh8Ph6AJ3Fh0Oh8PhcDgcDofD0QXuLDocDofD4XA4HA6HowvcWXQ4HA6Hw+FwOBwORxe4s+hwOBwOh8PhcDgcji5wZ9HhcDgcDofD4XA4HF3gzqLD4XA4HA6Hw+FwOLrAN+UPY6A25c9PSZSvzy2VQv03YfAg+6yt83VZWdMo/1izSzbWt9hnvcGEnFS5eEyhFKYmyQ3Lt0t6YoKcPCzHrvGjVzZJY3tn+Jf7xsemDZehaUnySnmd3LexMvxpbJE2ZLAcMzRbzi3Nl8e3Vcu/+3kc755QbPP02yVb7f0npg+X3coaL+ysk8UVDfZZgGOGZunvh8qfdI7X1jZLU0fv5vVg4K2yKf85o/LljJG5Uqx0GKC+rVOe3F4jf1vTt42fjyrOkjNH5kl5c7vctLJMSjNT5H+mlMjyqkZ5ZEu1lDW1hn/ZM6blpcv5o/OlSHnrt69t7RNfHmycpDx92fgi+c+mSnl+R63UtHaEvzm4GJ6eLJ+bNVIW7KqTF3bUSVZygpyl8wjYfHtHU5ucPDxHPjL1jc3wWzt3K7/WyK2rdoQ/iQ3eCpvyF6h8ggaPLs6WzKQ3+B8N/s0X1/eZJsdlpcp3jxxjdPR0WY29v3B0gSyubFB5WyFVb0JXaCtk9Fmj8mRDXYvcrLw1UEgYNEi+cMRISUkYLM8qTzy2tTr8zcHH2Xq/yINtja3yf2vLTR/wTNbXNcvd68tliI7lPZOKZV5hpuTpM0OP1Krs+tHLm2S7HhMrvFU25R+mcgq5dLrqi0gg3x9W+Y786gveO2mojM9OlaV6/L0bKtQeGma2wbNltXq+qvCv9g3o48tHjNrDW8jKgcD8oizTo2lDEkyPV7a0h785uEA2wQdHKR/839pd8prKkBn5GXLljOEmm9C90fjA5BKZlJtmzynW+sI35e87PLM4QMhKGmJGLE7J7IIMadv9upQ1ttkL5+Od44vlPROHmtDpLQpSksx4w6AoTE2UESpET1TD8hxVbslhR7Q3OFrHhYCZktv7ax9sJA4ebAL7fHUWp/bzOCbodZinWfocApyhymdCdpoZv5HAUP+kPrOLxhTIsLRkHWfv59XRO1w8tlDeoc4Qzz/gCV6js1LkXROK7DWkD/M+KiNFTh+RK0erMsOoJDBzrjqjc9SYizS694VidRJPGZ6rvFUguXr8QGKsGvUXK/1Nzkkz47i/kJs8RK6YWGyBifbdu2W0Otln6byN1usn63XhmXeMK7JxYAzxjBjb28cVGn84Zxw8wAvv1mfxngnF0qnOR8ATLR27TY9g0J5QkiM5+sx6Cwzgy/V8R6jcy1Z9NA55q7pjrvIFxuWbQVnJeJJAzLyizPCnAwPEAbL5NOXz8UqD/Qnm53id6xKV/1z3WHWYTxqWLWN0LphHxvBe1d0Y0DyjDtXtyB94iefoODhAlmOrvE+du+PVMWaeA75ALvKM+A45RfC5t0AvQNMz89NNz3Ae3k9UOddboC/eqTpshp4jU2lioDAyI1lOVdojwNGXOegrCIqcPiLPngNyhSAvzwa6T03YW5ZgM8Ez6IlLVdfDT474R8JVivDfb2nUtXXKH5dvl8YYZYlm5WdYBOuy8YVyx7pdcu3SbfYvkatXy+vlvNICMwIa23fLqpomaVajAOA8TlbnaawabmRIyAA2tHeaUzNSjWIUZm5yoty3sUJSVeHPVkZEgf1l1Q5p1t/AxBh3ZKgw/oIXWcgg+nPRmEJj+KrWdmnvfH3Pb5JUADOOdhXKATAOuZfgN2Q1MWbIYmJQMt7hKrAwMFHiJerANuk5AuNkStQ4MExxFBknQm52QaZUtLSbMcoM4EhjoDB+nLzguA69ZquOFeWN04zA5xzMES+c86qWDukujU5kfE5hluzUazy6tdqE+wcmD5WXK+ots8p9TFehz/U+PHWYCrkimwe+29rY2sWh7E9ggHxUDUOeRSzwkNIj2dVYAAXPPH9r3mijx/9Tfvj5q5uNJ3gNUuOAyCVO24Jd9RathxZRVNAZzv2YMD2kqGKEbgDfHam8VNPWYZHeEqULnL4N9S12njr9HMNvqp3jDZrihbFAIIcs23GqCOGtFVVNSk8JRvvQNtdqUj6FBgFK+QilWwxDxjNKX3lqxO8MVy0QyBmjx3I81+BfHDM7PGx8BvfBa4ReI1u/h895f4bSKxHj7UqvRIqZh8F6IPw/U3kxOBZeJMvB/QGug8FTkp4kE3WuOC/3xnmjkaQ8OFbH/1Gl95/pM0AGcU/Il/t1DslifXjKMJmrTgKyhawJtML8HK+/Yc7/umZXt/zWH+B5xsph2a7GKFUFsQLZ7LepTP7glBKj6asWbJR/rA3pipdVV8ArBFeQqWSuqJIBo/T5TsxJ12cdojPkIAY2lSvwDTSNPHtuR61lE6EHjD2qJfisTumCrA3GXECLvNA5yUrjZLSnK72hpzjnurpm0y38BtmLLEavBuB6ZBKgveBcg3U8LSo/oX1onIAd48DIxflCD9Uq/RLwIXhJ9jM4lrFxP/AdY7xwTIHJ+UodF7wR6LPJes3pSo8cwxzAf+gKZDhzRwYEvQR/cH6+R+/ifHRHvzwL9NRj26plg94zQd3UxMGyuKLRsrGfnzXC7uO3S7ZZNme18g76BT1DtmpFdVP4TP0L7hWaiRX+ub5cllQ2ht/1L5DLBfqcPjtzhDkpK3WOr160ZQ9fbKxrMbl72ogco5Ony2qlPmyTIP+xDwI6glaTEgbtqdA4e1S+0SCZ4mf0OOwheIt7e2lXndHcTH2+8FUgw5lrzouNBV0h38kuL9PnjYwtUjoOfgMPB7qC8cAX0GdA28hn5C9yWQ8Nfa96EQd0gvIXfA3fcA5sQWT6WOWV4H4Y+y69BsFsdNaRqiug1zXK1+WqgwL7iyBRpI7iNwHPME74j/srDf/LMVwzsEUjwb1hyzKHT22vkdyUIXKa6umspETLNDL3AdLVLv3y7JFytOo55hLd8uDm3mVsDxYIIGBfOnoPL0MNAwU7/46Fewy6/gSMTukpkUZKLL/y/DorsQuAAMEB+9mx44y5b1hRZgKQCM1fTp9iTIiwxCmrVAfoo0+stPI8DMhvzSs1QfWRx1eqgZpo5XYIl9PvXWQMS6QNZyMyW4fj8R91Lt/9yHJzfG48dYopdSJAM1SIYVggTP+ycqf8bslWMyxAqhoM3ztyjHxmxggTiGBzQ4tco0L7NjUgC1S43aznyk5OUAMjWR2dIbJNDa3PPr3GjAFK6Y5TgcF9AJwFDE4cNgTXF2eNNOXfsXu3LFSj6Hp1qO9VB+3rc0fJhaML7Rw6lZKsgvUbL643oYSTepoqjxtOmSSb9JliuGO0PLW9Vj7+5Ko9pbgIqWS9J54F5WsnDcuVZdWNcrUaxsz9d+aPln/pnFyrCh/jgTlhLqrVcGG4mTq+jzy+Sp7fWWvGTKxwOJehopxuOW2qOU44KN2VphCJ/MHRY2Wp0uC3Xtog61QBnleaL1cpHY7NTDV6gG7vXFcuH1G+QLFdps495TA4h598crVl8ilbxNiDpqpVwV2hiv3j04abYqZsDJpJUX67bulWuXHlDslVOvzKnFGm0FGV2Tr/ZCUxhinZ+7nSPEY2x5mjdMY0M6qT9APoe6kaGuf95zXjwUvHFliWaFY+GZzBxkfP76iTW1fvMIX+5EWzbQywFHQKT0JnX3p2nXxkaolVI2BMtClfPFVWI79avNWcBIJPn9B7UNPbnEeMd3iCuWzTc/zkmHHGc/VKr8gI/v3aC+v3mmfmjmANYydD8sOjxsolDy6RRRUNFin/lF776y9usDHBNxg+n3pqdfhoMQVMWSp0ijwNeLu/EdMyVJVFR+q9xebORJ2RYnXMSyz4d/RdL4c/fQPQ/O/1/nHm791QbksOkKVfPGKk0T4GIWMlcHCd0vst+ryXKP/MUwfm+UvnqpG92Rx+nL6vzh5lhhtBmnJ9tszpz48dvycgBn3gLN+zsVy++tw6o6dPq/yHXjYrf2GEA5y1m1VvXaOyA9pDp8GDV6hjFWTX+OzXSrs3riyTrao35qn++tPJkyxgAu0jY5eosf051Rc4PSwVIEgD0H8Ypjjt9yr//fmUKXZeZDo68aHNlfK+/65Q/krQuZlozjR6jHnAwcPhhm8xtv+o10QPUImTo7oKh+5zT681PUdwUodj12O84JrjxkuGnvd3qhswdH9/8kT7/J71FbJWz/0HfRYEU5CbGM44K+9V+fLZmSPlq6rrY1VydziXoZJBx3aA7x9RmwEbiZL8SBDYIuN+stpLJ979ijl/paoj0CE/Uh0CXcI72H7/2lAh31Z9wme/OH6CZcUe3VolP3lls9ykuh9bg3uDfwjY/u3MqRZkSFWa4BzIaYLNn31mjQUkTyzJkYcumCXP6JhKM0IBCmQ6wb3z/7PE6AZaxLmDv5Hp2Ubbg2wp1D3qeCO3sUevVP6ipHWH/o3TxhivVTsMW4lSSnguqDAJ9MVx/3xFzi3Ns0AG2U1oGEf2vY+uMLo/QecGuudeOIbAyBOqD9+tz5DgzQcnl9g8EUA1HaW6kPlgngkWBoAn4I35RZk6p+NkpdpQNywvs/NSAUTA8mNPrJJNyg/oAq6FrEDHEWSFV7GBP6fzFkt4GWrf4ZnFMIji3L5mpxlzYb+n3zA+O83KfYhqXr9suxp1IUYKgFAhq8d6qvs3VclqNSDz1fH7+5nTLAL6Bz3mi2rEs47vgtEFMk2FVq0qzkY1jPeVWbxImR+jmAzbOx5aZplUXhirJ6uzRHQJ45N1gjA/5yPjiUFJVoeadBgdYQgQNkStyAB9XI1wznWKGotWVqDSCcOXqBwRYQzzb6mRydgRGJ9UwwGh9OvXtpqjx7FcH0MbI+D3y7bZ9cluYEx87fn1ZoB8RY0ZznmXOgMYqawRGapzQrlDhd7XChVWCFSizGRccAY4/wNqANXonAazzFqba46bIF9TB+AMSk7VuMVJpCzr0nGFMh5jV+eAqP6zashT3vu9hRvlLr0ea0sZ51shs/iEGjtkL1Am/Q3mmhJfniFrjtYrX0Rjqyr2e1SxQ4MEHlC0H9NjiHldrE7N9Uu3m1JE2RNVvU9/R7ajp8wia614lgQ9CFL8WA1ty5IpvUAj45RXV6jxiFyABvgt48IIvlEVJ+bnpWqQU5FA9nuWGuw/VEOELMmVSp+cC8ORYAQRazKZRWo4sMYJen9YFTC0ffOqMsuefHveaBsXRgfGAms/yMpjSOOwoajL9DozC9LNcPnNa9vMCcD5PHNkvp0Ho4d1WxgoyIP69g6bN+bjKKVpjP2L7l9i43+lvGEv+mUdyTfnlcoXZo2UM0flWfSeyD1Bp3NULozUMeNEEjSBB+GHyMwkhsuRxZnmNN+iTnb/U00IscwsYsCxbpYAeyzuj9LfGXkZsr6+2bI33eFlpStkIUGHHJX/OHA4Zqwf/a7KLQzdFHWkCKQRMNtY32oBp31lFuHFC1XWYsy9Xx0vZHSD0jkVIWQtoB0cLJ45DhHZnUCvECw4UfkFHffizjr5ssptZOurFfV2rj8r7eWkDLEIf6KOi/VgnIM1k+iOv6/dZQYkjuCn9F7ItuPEYnjesGK70TOGNjxU0dxuvEIVCsbnX5QHyDKhX+44a5rRLLQOT92i3106tkim5KVZIIlgB7qCbOcdqse++9JG01EEWuALni9yCT17+xlTTfYepbxIMPVc1eEfnDw0XEWQZs4qcvKbL62XfyufrqttMQeVMX5y+gjTxw/o54drZvF+ddCpugiyZv0Jsm8EKrAtblY5g30RraPIkpFRvH31DgtkjFJHkedF0At74PPPKn1tqJQctY9OGZFjDhfP5iSl254yi8i6q48bb3OLbGc9Husa0S3IH7JmBB8ZyQd07otVfn5/4SaV8evkJeVRqmIoayVgAW+QMKBCA77md+gnnCj4gMw+sprMOAkCzvk+dfZ+/doWsxfRfccrX7xHHTzoHb57Tcd4idov8Oj9eo+ra5vMYSOoecUjy43PoPevzC4NjVF58Reqy2r1WvAiLwIc2EPoUJxybMkvPLfW7Dyc3CAxAL5/1Bj5iTqJ71LeJpBKlhP7EX7BSR2hOg3du0bHsVl5FgcbJ42/cTIJTKIfPbMY/4iNxXkIgMgL0UIMr/7GqMxkM+QQKq+YIb63swErosiIxrCIHqV+ujo0KOU7lWHvUCMNpiUC+u9NFTJcBSfGJFmRfYGF3mT9iCRzfPDiOmQphul5iLACjGMau5CZYAEyCpSxTlPh/H4VuLwwYDAEblqxY8+5XlQjg4gwUesAGNJkVjBg+Q3Ne67RMRC9RjgGxyKcUa5Eq3BoibARRcMYsDHq3xgzZFEwnDkGR+bHKrRRDGeOzJXzMDbCYNz8DuOHYyJ1Cdf6o94TAnqzGk4ISKKIlA5x7y/rXBEhwyCuVicTp5bxE4WMzALHEghXHPv+p9A3QKQ+iFr2N/RSkqcG5M6mUGlld+DZ8OzgC+iB0lMisjQyWKmKHpogMML7Y9Qhe9+kYilK63mNIcr/FaWRLz27VggsQAecAyWL04aBnaJyAUA+XPOWlaHoKor37vUVSqftltUncEIZ6EjlwxuVdnAMOdeTarAT2CH4QBMGjEf4jIw0mWtKm6Ar6OunyhPfW7BRjfU6OxYDBT5PJ7qr48AYxinFMILmoWvKssmQkJ25TZ0CeOJhVb4Y+YDsEuV8IJg/zk0wJzorTnT+98oXOORVakj/Vx1ojAmc5416HPf0vy+ss0g8hn904waMbsqlcMIj2K1fgXFFxitWINiVkTjEHOJYAHkKC+6rOQUBBIwunB/44TSlM3joRaWtF/SZQmMEFnYqzVAmfKzK0TcDBhzBE6pJoBder6qDCO+lJ5JtoGojNAlkFMjmB79D17So3CdwyE/gDbJ06BC+X6E6gKwiJZo4ggQ7AiCvWYoBfXJPlBZ+Xx3eG9VJ5Fj4nGvxHbqR7CHGN+fnBV+wxpYqGpxWMk8498GxOMI4u1SKGJRQkQNckww+50X/BvqCYAt0/wPVFehE5AIVLj/QMTEO9ANO6A9VD6HPoH2cWdYx/vSYcfIzfXEtnHnGEAvAD/BFLAE9BNnX/gaZLuQoDj3B5e5KI6EFaBU6wsEhc07wHF6Bpgl2IHNZZsHx9EfIULreFzgWuwEZiA4I0Xq9/H3NLls/TKA+qPrp1GtC72Tx+R3XIYBGyShOE84YZfzwLHYGsvcJ1RWcFznNOkMymqC5s9NoHH7hfrjmX1btlM88s8ZsMM7PiyAH9hKltQRL+Jt5YJ6Q+yyxoUSaDD0BlMCOonLt6e21xosEz0kKIOdY/4k+4HfQdGRgEOAoYxsybnQJCYFvvLDedA86DRuPpoFUABHEJ+CKo32bOvDw+UAAGsVJdfQNPmNhoPQwVGNhdCBMMMBhYAyASCemO7DeifpyjGkiyAgNQIQYRcq/RFkDwdITVtc0mwGA8UnWhOgzGQSEaKD0A+CoLVcDAAFDFAuBt0aPz1AhQ3SMF44EypffBUAgUr6E44rAYToR5rta2mztFOdCODIODHPmgjHwQqGm7UPZDFEG57wo66CDJUIe5c+cEMVingIQ6d7RQ6dLjNwHNleqkN9p94qBjZNBJJso4VK9J75HmXANHAPuYyBBua01sOh/Et0Dop8Yq7HAIP2P8kqUHMbem4GAC0oRBYbCDPgImlysjsxgJT4ya0SFewLXgnbIZKDYyRJAi5ThEBWOVCo4aBjbKGqcNJRjKGLaYrROyQ58SGT9ng3l5mwBAh3/3Vplype1KThT8AH0xDUDBczvcMIoKaeEjJJPyucwLvb10DkfJbEYuRgTGC1kv4iSYyyxroxSIoARwmc9gWMwVoioV6oRjNFP1Bv+ILOPU/A3NYwwrKP5gbIlqg8wqDCMY7XCgcxArAIagIAiGX6CN7FAKIAZWjvbG2AMjc5MNRrdorTJM0dHUD6LMUlmO0RT+wbPHFqgeoIMFevDaHw2VOktEvDqKqUbytgCbKhXo1XpH9pk9PAHZXrQKIA0OD9VMxjYZNoD4GzBC4DfYUQzDoKOZIU+r/x5idIa2Z+eAM/D+8wYpayBYcoUonfgOdYb0+yK31DJg5MZbQwD5D98/ve1O+08BGWQNwRUyPxznxja6A6M6gBUxvAd90OFBmNi+ce+xn2wgKO4L7nXHyCAQhAhFoD1CBIgk0P53zcHyxywpXBiXtXnhP2FTcKzhS7JpiW+iRxBpuNYEfwjuP7+SUOtVJv1qGlRepLA92PqRMGHgCVFZOkAYyErStYa/YEDCJ9Ch+gv3uNcBQ2roEHGyDkB9gt0TLASp5kMJXrrsnGFFljqSTJhI1LBQsCbclR4CqzWaz67o8bkKMs0mCfmFvqFppmr7oC9R1aXOeS36L2/qk0FX6AHsKnQafxNppHgEeNGh5IsGQjgKKfFiE4PJ/iMhYGTCHNiHPc3Qsbw63ZNDPGeGJuSBhQ/L9ZxITA6I+Qi56GlPUxHNDnIHvQEDBwyH3SCpKyMOnmMUQzYaGMLQzA6ks2YkVWMhRdjaY8yCCm92aWCASeQhhs4oYwzukSEcaDMMTC/NHuUdcCkhGFf888IURAodOYiEkTRMcAj74OoGgK4OzB+BBflCBgyOB5TctPkGHWcEeAYMAhzPosX4BDgjMQyt4gxhUMWC1DSCZ2wzq8nQ5zvoC2+JTuAQURzGRz+yHYULUqcGAKUg+Kk7QvwGZ0koUcaG1F+R/c7yuL+v737gJPrqu/+/9ve+662qHdZkmVZlnsBbJptwGBjTKgGU0NLIT3hISEPrycJJAQSAqEaEwcDf3oxNsa4d8mWJVm9a3e1q+296n8+Z+fI1zO7q7a72vJ9y8f3zp27M3dmbjm/e1r05lG40REu2GCW/ZEgl+OL/YrOPsggst+DfZBqN1Thy3fbTukwn7N74KX7Jn/PjRu246MuY041pqtnF/nS09E+AlWS2Sc4JqKHIwEC3ws3YMLNIEpG+V5eejS+iIwsXf5TzZWq73y/lJpyN5rjhOPiqqoCX+UoHKv8HhxHZOJpY0z1bDLSI73HWKNN9lB7n4lB5y4MUzRRASr7HUdH2ig7Ab8B+yD4rbk50N7Xn5DJI7PLPhvWHQ0ZWm4q0tnaX62b768VlDxwXESxv3ODMHoTgswo7x1Kt5hyPMQHYzxmvXAzA1x7wo0IPjHVOzkueH/ab1GNjkSmdyQc82S0OVYp4Qk4R3ATkY5o2FfDZ+HYGan6JMcX67GPUyWQDrXm52X445S2X5T4Uy2VbSQPEZAppmSFdoqf23jQ3wCiExCut+ONNtZUEZ5InB/oXG4isFez3/A7828kXEf4nTkmOF5DfiSK/FON23fZB090VHDOI+ihhhN5qNtWVvp9kTwV15EomhNRLTfsy9GAj/4kqOLP9nGeZt2A7at2ASTXnnDt4npBU6OwHvlGOmN7xexCHygy3Blt1qniynbQbGg47LsES13uWsS2hHwZN8gPu2sUtQE4bshzsp3x31U8Su75Ligp5bOE6xd5Oa4dlCKSz+L6QRDKeeo722rPWu0s0Jldqfv+5dSc+IoxQ3Bw0GCag3i8HWzr8YEMB1M4yIbDhYmDjZMG5wiO/+iazPMcJ0AO6egJZzjUxadx/reuOcdX2fzGC9X25t9sti9vrk4IDHnthK2KLeBtwlvFr0P+ie3hojzc9rA+F99/ddtBA3DuWFO19rU/f87fhRqt1AO8ot+2uDcOj0f/Bl5ECdKXrlpm97htIHClrdavXrfGvuO+GzLCNywo9dtI3f7JgiCW6lxhPM6JQEaNoHkicPOBKl0ETSOVZpa4zAhtS0OgQukVvz37XHRv5CHpBIeEvxgvc8fZPW84z379uvP8MUe1y7fcs8WX8MXfbHjxHV7EsrC/+/3TLeA4COsypVbA0HrD76P8pJTW3/Wa1W4/PM/3UEeVcdp3cad2tGP7eJAc3jCG7QjH4smixOa/Xr7c/vaCBT4jwHHBMUEtBPY9qkb9/Lo1PpPEjQvOP5zDfvuGtVbmMou0cWaM14lEZiQ+gBlP7JuMX0vGZyJw9588Gzc+hsPPTokVJWVcx8DuQoYxPiPNPoET7RHsN2RAf/Ta1f48SObuPb/b5ju8+n114hiGcbuef+/o0rD7JmxPLI20PRyfnzhvjm+v/+kLF/rmEi//yUafqPkxmpFek5JaviW26UTXTHDThE60fnrtucaQMOxvdIZ1xytX+g6llhRkumB6nn39FSt8AMG5icAkZPT5/Z5vbPcdrnEOn4iSRWrZ0IZzIq10587QedF4I1AkuOF75uZb+K6j2IfJz13ofi9/Uz62TvyaPB46T54YeQM6fPrF9Wvs3csrjY7z2BfpvGy4vAvn9ej7sp1Mj+eP3H+xp4/j4Ym2h2YHjCt6t8u/fHzNHF9b7MMPbPft5GnnGL1BEsX1kmfYpuh3xizvyUxss07KH7ljk++DYTC4ZtGhIXkqauacV5pjP7vuXPvG1Sv8OWS9u35Q82W/C4TpMJG2ynw/XEMoBIhszrjiZg03wuTUKFiM4aRDaQJVdMYbVYGoJ061BTJn4QIfj2o/X7pyqW/ITbUDqnlEA0suSpTIcUeP4JOTxGi4yHH3jMGTX/nTZ33DcKoGDYdSrFAFIshM5s4cVSB6fGJb4u9Q065yQV6Gv1tHm8H4u7W85sfPne3vqtEo+7pfPOcbilMn/0SobkKpCFUk4oMJfrdwl+5kPHmk1ffKSqcmdOrzZ4/utpvu3mxfdRldqt6+53cv+IG36RxnsqAKLh0IpU7QSRWrirkjODGZDjIAVBmjQxfu/g2HEi86kPmhy8QSNFOiwZ1j2u1Fvxba+FEqSin3aFVa6bDm3cvL/X5zq/vN3+qCxJEG/efiyg2eaLtmllHaxrZTvanWHYOcS3xPp7F9NAztwb5PyeRwVd3m5FAtKdtd5Af9Rf+W32zxVe9OBtVOyXiwbdELbmH6UHftrZFhPU7ku9uP+B6BqXr6YE2L732T44AbOrRPYZ5ED6xUyeOu+jdcJpmq6B93GRXaNk40Snf4nBOFaq904MB0ItD9PudbMqrDcr85na1822XK6LCLjGJrX78v5SYjHcU5hPMkJQsjYR+ipIySC9rocS68+Tebj1chjcc1Jf59OEdTJZHSTbDPc85mm6JYj7/lvD4stqUg27d5+qvHdtt7XcB6MigV4drGsUhmNCBDzHirdJBS393r2yafCOux/9PDJJ3R/f1T+/wx8M7fbvXfyde21rjvZ4vvNIq2WmSW//6iBe6YP3u1UsgMn0y71LHEdWK8x0QODrrv/YubDvmaTZT4RpufBNz4+9DqKvvyy5b5m9IlmdSGOnb8RmNQ4fYFglz2wRFirOO4QcRNNNpy8/uzDSNhX6ODvjCWb647X3BjjZvl5NVoA9jjzvecp6OBG9tHyVwY4mY4/L4EjFSHvv4Xm+wfntp7vJf60fC+VLfmWGRbQt6T6uJ8X3w+el0d7fwQRft6ev2lWin5uJf/dKN9ibb/Lk/FcXHxD5+xG9xxQdtHbjbSodZTb17vE2P0co4h0PzRtasTqrePFzoi4ruTUzN8lDIDkfGkut0FbocerXrLWCBjyZ0gTnh0b0xQyF36gJIVukKn8fPWpk7f+JluizmZ0REAVebAgU2PWCCztq9t9AbDVE+gsw4yxtSJpzSRzN7VcxIHc6YKKYEf1Wo4kdFD6JVVBb7zEbpPJlG9gg476Nk1oKpNjjs5kpGkV8r4W1S8Fr1zcbKiFIn1+Fx0b02Db05aI+HkyZ0pTnIh48R2f8QFn2TGOXHSbuRkcLc3tCUgU7HLZRo4kXJuJtClx0vaDoR1zjZO5PRmyb4Z7lROBKqskHHkrmD0gjYeuIj904b9PvB514oK33tiFHf1P7J6jr87T5f5ZNRoC0Fm9bp5xce379p5JX6/puOZb/kbIiN3DEKnDFSdoioZwR7VR5mnd11urkSrFpHJpp3KDQtL/O9BxptjhyCP9sBPuwvm9ljmk2EC6HUVZKBucZ+F/YqOAHiPeGRoeV/aSJLJqHbBAccFvdq9fn6JXzYSjiHaKNLmlxteBDGUBnO8cjzdc7DBV2E6GbSZ4biluipVWDkGqD7FOYv34DGJ57jRRZsdltOjK+1XRsz0jwNuEtDDLMc+8xOF34heqS8qLzj+G48n9qvH3Hc7y70XpVnRwJgbhbRVeoM7B9OmkWOCcxtBHudIak1wXeOaQmdP7Iu0P6dnxtEQZJGRZJ/nvMsNNPYJqqFS1SyK8xHjnzJkSkC1M44PxrvjEkC7YNpWhSqY7M90QENPiZzT6ZhpOOz1IeCjqhxtGTnOP7l2ns98joT981vbavyNIgJ7qnODDM+rYr0gsm2Mt3giQ1XGh6p4d7prJz1rcgzQHpQYmc7b6IiKm8Ccw9ym+t55Q5VUbmT6AfldBp+bxFsbx7eTG35z2oDy/U8krsUrXYBMzY/xxr7OuX9zY7vPN7zD7UtUlw/4Dv5g6Szfqyg3FJ89SqdfTX7/YR8Mw7BQI4Egl+sqPQn7PMsoqCHCNYN9kH4T2M8INOnhk2Msin2WTnNCfpL2ggzlQQDIfkCAtcedu/nO6JmTY5kgkFoLtJ+ks6X4DsQCjs2ha1OSPzb5LvzncvsZ+xvbORzOD1wv6OiGbeEGI7i+U3rO+9Eh1IkKHgLel3wgeUo+D/s2N4bofZVjns/JjZ7/3HzIPvbQDt/D8T8+s8+np+tafR8AtKX+8vOHfVOi8UTBBh3scG7iBq6cGg2dEcOxxQmAi9hQQ/yhnjnHCxdeLn54tcvwkBHgYKeNCHfK37So1DeOppSDTBjrcrHirggnJ8ZJ484hddS5m08vbH3uPEd9cUo6aEhN5jM6dMYx9yk5YXFC4sTGe73KZS45mfHZuSDS+2kYlJaMAiUeZMZoL0ImhN7f6HaakwlVCZYX5vgTDZkmXo+BVmnczJhTlKLQ7TQnNk4kLCdjwcV0VVGur1bFe1/uPgPVYimJ4a4ameqHapr99lAfvshtKyde7grSyQaZoBJ3QlyYl+XvKvou2evbfccblGZykSZY4PtjGIbowNBRvO57V1T6BtxknqgKwXYwaO29h5r8hSAe3y13MQmkJ2roDC4KdEf95sWz/G83wnVgXHA8hAsPnZqQuRqvo4LjjbZ93NDg7i37HRdQ9isSQ79QrYh9lF7luECxafluP+I34QJIxoHfnuCBHuoYK5P9ZaShM9hvuIvK78nxxTHIMUSGmGOUEnC6I6dHOTIj5dkZfj8hWGC/J4AnQKJbcW5W0BaK44IqpezLZFrICHCnmFI73p/2sVSD5rjguCS/yvHHRZ7AnEwexynBKo/5/dmnfSdT7sJLR1wMS8B28AUQrPFd+My4ez9KWXl/hgzhgs13RYaCY4X9h05sGA5mpN+RGz7nuO+h2v0W9CrLzSnaMZKR4I46ePy+lZX+u6IjD44VMl3ht+LOLcc/pZ4nkR8/ZbRd5TuhChbnuImqEgr3Ux2/VtA5BdXhxnO4ADJfBOy0+2PYEkriyJDzPVPT4JYl5bbfBT7s61Sd5iYgf8Nvzf7K3Xv2f4J7Ap0f7z3q91XOZcMNnUHmjmsONxpoD8pyvmvej+qNvt2T+763uHMrxyr7OFVg2S7aybHexS6RkfyBOy4IBjlDcq4nc06iIyQytgx/Qydt0aEzuEFKx09cB6i0x/7Mvj10XNCO0h137vVZn0COcwYdyLB/8zzHHZ//geoW32smbcxIPMf1iWObIRV+eYBr5jEfTII2hhxfw6EqKp37cHzxHVMqxPmJG1gMA8J3FjogIuDg2KUmENdqboTe4NYjo851kY59+D3HGudCjgOG8uEznajDu7E2dK0Y6ojp2YY2nx8Zv6NiCL8f53ryAtwg41zP/scwPwQF7EOcY+mRmv2RmwXsN5e7fYhjg3Mb53w682L8UYIbztfs85wz44fOoJSfcz+/K8eP7+fAvd+6slz/mGsBBQE0X+CGJ3kJrgdsG82AyOvQ+Rd5KKpjshewX/J6HGPs1xeU5fvzKTdQeC1ujnO+Zd+kwz1ek/wQ+SeuW3TKs86dezkX+I4Q3XPk3Qj66D2bm0sM6cRvw01wbqKyn3C8cKywr9LbPte6O9x73r2/0d+k5LgOecKRcD3mGsB1+tvu++MaSxMeqphyjPF9guOF747vMCRutnEji+OB3ubHM7/N98FN5g+vnuO/4/gaDnJiChbjUP2Nk30otRgvHFQcPNxV4U4SJwpKK0icqLhYUsVhw9GhO8ActJy4CBI5QXGh4sJNAEabQ6ac0DgR0r6EQIkSGg4KxuthDCTW4YLO8Ber3cWT9yJzS0DFHTcusHe79ThRcDeOzhC4yJJx52Dj5BW9A8xBzt05TmRh2ynhvHNHnb94kxmmPRgnLO6Oc+eVEwKlMNyF5MRORo/Pzgnu1+4kxUmS7SCAI5NOlUSe50TMyZuqcJz02EYy73wf3Pmma3UyPdw98kGoO0GyDVQbIqM/HEqkyESFHmKpLsJ78T3x/Q8XZJL5ILNPRp0qrGzvyfZSeKq4+PO9czEjIOYixuOJlucyPWQcGXCagOFEd1/PFN38k/EgwKJRfNi3QIBOW77wlbPfEpTwe5MJZD32JW6gcAFiPYbOIIPLuuwjZOLIgHJj4WH3mMwppd38Lfs7mQEyGXe6YJMbLmQEGVeQ6kNU0SEjQKaY/ZcgjYskY6KyL5AodeCYYfs5TtlXyHyzj9KDKjeGOFbJ1N/tLuhsIxmAHjdDJyb0qhrOB1TfoadWMuOck8h4d7rvn9JQqtKxN9BDMsci7bkYsoa/5cK4y+37jK/HcUCQT4aESIcLNXfkR9prl7nzA8fRs+44oK0VmS4+O98TxwbIaPCdczyT+LzhdyJxjGw82uG/x7HOBLBv8NkJiD/qLv6cgyaytD3g/MtddfYrMmDjieOOcyuZPDKEZCD5ngnA2He++PwhX3JCuymuFXTiwtdORpFMH1OO2i9tYniYoZ5u+d54nYdruKnW6fd1OkmjKjg36zgPkxGP/q73HW6yTe7czDFErYuj3b3uPJ/im0GwjWQcWY9SeoZ44njl1+eYIKAm+CRzzf4Cho7hxiY9XfPZOJ8zJAvbE9oLE1BSRS3sY5S60LEbn5XrFzfsKBXi83Bd8f0OuP3hfretHO9kxvnO+FuOSa5FBBDcwOQ9uZaw7QwxwmsNpyDDfe8uIOhwERDrcW1g/yeDzfWO6yW4ocoxwve4rDDLHzt89wS1XM8537DNY429n6CBmwi0peOm2dm4VhS6757fgdIrvtPxujaCl+acRD6N8ynfddhPuRnOtf+H7tzJ+ZMbur42ka+xMeC/H86TBNTsXz9y+yqdcrG9dCjGa5MvooSe12NfZJ5eVEPpJDdjeI6b3OxrnNvJ67BNXCfYXwmY2P8I6tgnyPN9dsPQ2LtsB+d3/o5aIbwv+TfOJQSu3JijRgj5QW6EcrPwOXfu5rPwmbnW8Jn5O/KENI8gOCPfxI11bmoQYPJZ2BepvcZ1gnzkoY5uW+7O81SVJpDlnMqYrAyrxvmDayDHBsc4y0fCtYz9m8/A98d7cSOUwgTen/P/SLgR6n9D951wzR8vHAZ899xkomYS3zGfV05N0rGJ6t98CuEk94Pddfbnj+72GTB2aJGJxsWeNj1/Hustlovb2cIFigvPB36/3VcfIfOjw0ImGpd4bupQ3fZja+bYVZVDVfLPFu6YM14ZGUCuFbqaytlA6RU39P52/Xy7bn7phPeEGsUNNG6+0gacoJE2tDos5Gyh9JSbA+SjKEmOb18tJ0cli8OgjQR3H7jjwh1RTn7jWc1IJB7VTLhz/cm1c+31C8psrpunRPVs4b25a8+dVEpyaa/DHUiRicSFn/bT9DpJSfLZHlyZ0jiOTXoC5I4+x4RuLspEIlBcUZhjf+YywwSKBI1n81pBKTE3OVcX5/oaVJSCjXdTDZHhEBjS1Ir2mS+vogr/xPW6Ot2oZHEEVJ2iehvVEamzTccUtE8RGU/cqCBIpPMGOg+iWhdt7KjSOxlQHYcBuO891Oh7zNxQ3+4yxzqFyPihhJ0bFee4DPGr5xX5iz7Vp6iaPhmQEaadO1W8qLr8SM1Q+yiR8URGmNomVOulLSvtTQkUudF4tnFFoP0e1S5pn0fzkVCFXWQ8caOEm4pUzaVjK8aipGosTSPk9ClYPAnUvabu+WZ3sqPaEe0ayByoZOUEujvNNj5gxrAYKSku1+cO1pRUl9w0lalLYdnxafT5yJTnznIpwnggI0wHJkWZqVaeme672qZ9DW0paa9Hm6HJeCOMNhe02WQIEkoZOS7ofIh2rnICB7abHa0xS88wy8k3y3YpJ88sM2foOJjhuPNLm0naWzP8CW1+aN9CG2WGPaiaRDdP4tFWlk4oaO8UrhUM3zKebbemlEF3zeztNmtrGpq2t5h1uiAizR0LS851P3zmtDzPjwWuFbQjo7MdOgahrSXXCErYKcWjreBkRPvZ+921gjaiXCtoa8rNlDBY/bTDPj5Ie1t3zPe7zzjgUl+PS1wb3TIek3rdsv7YMvpV4Hjo7x36W7LlPe5xxVyX5pvlDfXgK4nIH9Fet8Dt/1wv6AiLds10QEf1U9pqqurpmVOweJIoUaEnORrf0/kK1VNDA3wZXveRatvwN7dZf5cLGtPShzICx5PLHPgUfTzyNCmNlOZizhQXP6b6qZ9nGnkc/3z841QXdCa7E0vyJMmQUIWIiz+licvdxZ9ey+iIhJKUyY52i3SUQcN/jgnaqdD4X0bG6Xb39/7bqh++dyhALKkwKyp303JLKSqzjLxCy87JtqysLMvOzvaJefbhmYKCEToFovOJJflZvpMgeqGc6N4dTxedPdDr4LP17lrR0O47nRiP3i8nq4GBAevtdQGySz09PdbZ2Wnt7e1+2uuuBYMEiEcOmnW0Dt00aa63zKISm//2j1rJvIWWmuHO+5KAGySUjtChCGNPnl+W63sBDeP4TWbcLKnv6vU9anNM0KkJHctMFpyX2W8HXNDW7wI5Px+XWGfQBYLs1zwOUwK7/sh6Ay4IHCQQJAhkSupqdxkietqNLSN47OoYuqHul7nvotMdDzwmgCRb7o6P0itfY5WveL0VrVjDZsowCBYJBgkSKWlfWZTt81FcL7jBImNDwaKMm71799oVV1xhTU1D3e2HXY1pdP5kpgR36enplpuba3l5eSedhls/JyfHv1boQZFpdP50piIno7+/3z75yU/aV7/6VZ/xYP9mSiIwnD17tq1YscKWL19+fLpo0SIrLCz0xwCJfS5Mtf/J2cK+G/bf6Hxra6vV1dVZTU2NHTp0yHbu3GnPPvusbdu2zWpra627u/sl+y/zixcvtr/+67+26667zoqKVIoiJ8b+FoT54abx68WvA87LbW1t/qZGmMbPh5sfR44c8cvZxzs6OnyAGF2PxD4ehHP0cNMwj+Eev+lNb7LbbrvNrr766thSkbNDwaKMG06s27dv9ydSTqqcRKPTaBrpufC4q6vL+vr6fOlgyDTHp5Gei19OKU1mZqbPnBNMEjxGp9Hl0efiH4d1eX2Rk8HplmNi06ZNtmfPHp92797tp2Q+yLRkZGT4mxlMSexrBJFkqEkEj2FKqSP7tMhEIoPMeXnfvn3+pmB1dbUdPnzYtm7d6gNCbhCGczaZbDLPTMkAFxQU2MKFC23BggVWXl7u51euXOkTj9PSVBVbRsf+R0k1Ghsb/b5GfuPo0aMJyxoaGo4va2lpOZ634O+Zso9y3iWvEW7cDZc4d5NYl8dhCqZsE1P2XxL7esgzcAMkPz//+DLyDfHLSkpK/E1Bzvmc08vKyvw6HBPFxcX+fUTOFgWLMq44gXIyDokTbPTxSMvil4e7elwAyHhEpyHxeLhl0SmJ1wJBYzixR9OpLCdTT+AZEpl30nDz0WXhcfx6XDhkemM/JmNCxoVSGKYkMjpkukMi882UzA77GZlsEhmMMF9VVeUTwWR0nv1J+5KcqVCCQvBHELh//36/n7L84MGDfv8lkelmnyZDTpaC/Y/9lAxwRUWFDwjD48rKSj9PItNMbQ8yyfwN51Ttt9NXuBaT2Gc474FzYvwyrtPsZ+xfPBeWMR+qNoPHIXDj9UdaxnuQovmKsA77XLiJHL0uRx+H6z/nXR4zJbDjZnFYh79hn2aKkE8INwBHW8bfh9cDr8N28Vg3pOVsU7AoUwIndE7s4aISptH5U3mOaQhCh0ujPRdSuNhwEeEkH0724cIRvyz+ueHW5eIxWuKiMtzykMLzTJXpmlrI1BAYUs0pJDLoZNbr6+t9Rjw+kfmeNWuWv/tMpjzMc1e6tLTU35EOibvUIRMjEnAuI/NNQMj+R4adfY5MOoEh+yEZeJZx84L9jn21ubnZl4iwn4VSEvY/9rWwjCCQZXPmzPFBIeuwTKaGcJ1jyrWO7CL7R7h+hmXsPyyLrh+9hoZ51gnXYPYh9jHwfFhGgAjWDzch+JuwLLxPyLqGmwwEVOyPoy3jGhu9hoZ1uIZTUyh6XQ7zIbEOiX2Yx+zPvAalgPHr87oi04mCRZlxCDy54HCBC4mL0nDzJ1rGxS1cNClFjaZTXcZ2cUHjjiIpVFcJ02iKLhvueRIXtnBXMlwQo4+HWzbcOlwMFXiePeyrlOJQqkOi6l+Ysg+G/TDsl8wTLM6fP99X9YtOWc6+Ed3PmCfTI9MXl3nOMewbpHA+Y98iEORmRKhSSsBIVWluVJBx57xERpp9hHMN+wspBIJUiZ47d66/WUEbW25gsL4yzBOP3zj81iGQC4EVy7hWIbqMeaYk8HfhbwnUwr4S/oYbCCxnP2L/YRn7TFgWzkPhXBSdj15XuNaE8050Wdhvoo+ZD8tI/B37GNgPw41W9r34ZaEKJ8sI9KLnPlK45rFcRIanYFHkDHCB5UJIporEBTPMc9d0uOXR56LLmSdjhuhhGeZPZUqAFzr04aIa5klcMIdbHk3RdbiIhot1NGgM86c6lTPHb8y+t2PHDt9xCIkMPmnXrl0+g8c67E9hym9IsBjfgc7SpUt91cBwUyB+KlMHv3VI/Obh9ydI4BxDZzOkLVu2HG9fSCJgZJ3ob0+ilISS6rVr1/p9ZNWqVbZkyRIfGPKY80M4N8jp4zc62Wn8MoTl/Mb85lxfKClmGR0Nhd+fUmGW8ZsTELIeNwq49pA4p4T1EK5J0fMA8/HnheiykabsT+wv4bpD8Ma+BYI2locbEGEZJdQkngvLwmuEZSIy/hQsipyhkCnjQhvmT2cZiUw+F2im0RRdxjwBavyyME/iLm7I8MUnMncnWhZ9zN1ZLtKUKnCBjs5Hp9EUXRad57V4TTlznLq528/dfVKYJ7NHSWS08xwSPVOy30SrS/F78LuQIYt2nMOUkkiCApkaqMZH1dBQbZmSQm4iEBzS/pX9gn2EFKriEzCwD1BllH2A35tSaIJC9gMy7hy/ofpe2GcojQklMnJm+C2i526O0ZHmSSHAYxnXDn7P8Dy/cbi+MEUoJQzXmOgy1mM+XIcQ/dtQCsc5gt+ewC/sEyxjv2AZ+w4BYDjPh7+JzjONv76wDyG6PCzjdcM+xnMI64UkIhNDwaLIJMGhyIWbu8CjpROtw/NkIAgYSaEtSPw0zMc/jn+O1+OCTWYxJC7i0cejpei6ZDjIYJAJITEf//hkn+N1w11reREZPW4eUFoUEhlMqopRohDfiQ7L2W9CpzkhcZefksj4DnRIlDyHTJ1MLI5JggV+SwJDAkSqj/KY35LfOwQVlC7xPMczvxfBIInMPb8tgSFTqurxm5KhZ8pj9gEeKyB8qXBuDedHcLyRossI3AjioutGU1gePW+PNh+m4f2ZDwFfeJ7finMi50gCOnAjgOUsI2jjeY7tsIzfO5xrCcCi6xEAcnOAc3fYD9gnwvk8+n5hWfR8Hz8vIlOTgkWRaYgMBCUHIWMSSp/C47AsOh1uPjwmgxKfQsblVJaTsSDjEDInIXgM05HmR1pGRiaaeO34ZSEN91xYFjJC0xm/AYEEQUW0Ax2mBB0EFqHjHBKZXX6v0HkO0zBP6QLtgwgqoolMozKFZ46An0AjBHu0E2Oe3zAEiSSWExRSksxvxnEfSnJCsB+CwPC7heCQRFtDgkKOp+n6uxFE8X2GoIrvkGCLafiewXmO58N6LOf8xzrR9UIJXzg/gufC+tFlrBc9l5LiHxOgxZ+PhpuGec57/L7gN4uuR3DHuYzfk98c/L7RZRzT7BNhGUElr0liW6J/yzzri8jMpmBRREZFZitkkEghExT/OH55/HNk0MgckRELGbBoCpm6k11ORongJGSOmYYUHkeXMx/umsc/R0aJzBOJ1w3z8Wmk58LyqVg9iswrJVLxnecQgFCCxe8Wfs8wT2aTHi6pqho60CERjITghBR+HzKyU+17GW8hKOGY4Hvld6B0KnzHBPH8BqFKMb8JyykR5m/Dvsx3GzL8lBgSABLIM6U9KkEiAQOPpwo+33DHfHyKrkfwx2OyNNF1OPeEwDDswwTWTFnOvg+CbpaF9UJHLpx/ouuFgDIc+yBYi54bhls2UuJ3DMdJNEWXRef5rUNHLiGwiz6v40xExpqCRRGZMGSMyYBRahVS/GMybcMtj08hc4hwGjvRFPHLuHMeMtzcmSfYYUoKy0KKPkeKf57MO5n2cDf+VKcYbtnZwPdL5xh0mhM60Qkd6VBCSSY6ZM7DlGCFMfXoPCckOtGh6ioZWj4TmdnodCYI30/0uwoBCW1JaVdIx0SbNm3y85QaEuTw/US/KxL7IEH5smXL/ED2BIPr1q2zefPm+SCC7/ls4DNFpxhp2YnWZd/jGA/nAKYjzfM9MR86cgnnmOg6LA/CPhemCN9tMNzjgIAsHPsksN9TUhtdFkrew7Lo30TnCRhFRCYzBYsiMmE43ZBZ5u4902iKLot/frjHlMSQ4SazGDKN8elklvMaZCbJtJEpD2m0x/HPhWWUJITAk8CRFOZDqUD88vj5kMj0UzpxtvBbERCS+Y5PBDi0k4vvRIfvku8iVGsLiYwzJY90mkIwGTrSoS0k39l0xj5GaSAlg9GSW5ZRWkVAQ3DE98o+zTzfPb9/KLGlB1JKCalCSnBIOzT2Gb5bvr+wr4T9cqKxzeFYZMrxyTwleOE447iNLouuH9YJ87zecMd9SMMtZ1/leyNFn+d4DCV84dhif+R4ZBlTlhHYhWU85rsmqAtVPHkM1uF7D8c9wutHl/E4LIs+F00sFxGZ7BQsisiURIaQIC8+kWkcbvloiUx6SKFKINMwf7LLQyaXDOaZpBB0Us0sJAICUpgfaflw65DInI4Vgm1KgEMHOiHRni50nhNNfD9kvKkOGU1k0KnOyjAMoRMd5plOFexvBDm0LaRdIaWCBNMkllGVlynfVzTxd/zWfC+0LyQIpHSQ7yP0OEkpIt8Tz5P4TQlW2DfO5PfkvdlXEfbjUAoXXRZuqpBN4DHbzbLwt3xejkMCXdYLxx7rh+OKKcvjl4Xlw83z2cJ+G78fj5T4fjhuSNHlfI/h++I5vnOWM6XEMBxvvA/TsIzH4e8I6ngM1hnLY0lEZLJTsCgiM1rIxIZSs5BRDvPDLRvteRKZaRKvG+ajabjl8cvIhIdMMyUZIfMb5odbNtzjME/Gl0ww05HSyT4/EoIigqVo5zlMCZgoRaNUKaTQYUvoOCfakQ4BI4FkNFGaRuBEZv1sIJghIGK7SQSIVCMl8dlCZzNMqRJJIoDmJgIlVQQzbD/zBINUXQxVFVlOyVUImFnG7xbFfhr2jbCvnGgabmCEv40uZ/sR9lk+G2m0ZQSUPOZ1wOdjP+X1Q1aCz0dwxX7Cfhddxv4TXcZnjO5XTNnnmQ8lpyGFfTk+heV8h7wH+0f88yrBExE5fQoWRUTGCKWKZMZDdbroNKT45SM9HzL6ZMajiaAlftlIKawLMuBk0EO1u+j0ZOajy8iQkwEfLYUSGBJBBiVtdNYSnRJMhsAlTAm62WaqX0Y7z2GeTlpCBzqhlC2UPPGeY4HvnG1ge0iUpLE9oeT0wIEDfvsJEKl6y2OCZL5rtiW6XZRSEfgRHBIIEgSz/bTjJFgMpV68J0LpWvj9QuIyzZTvMewnIYXvbbhlIahlv4w+z+cLAV/09wqBeFhG8BZdxuMQ0IFt5zMSlPF5wediHb4DPivPExRHl4X1+PwsC4nXCO/Da4qIyNmnYFFEZBIiOCDTH6oHUmrF/OkkAtAQdCB+ipN5LqDqZOikgykBQFgWEsvC89FlYZ7AgICFDnNCCoPIE4AN9/4EFASO8R3oEEhSskQQQ3ASn+LxevGJgIopJYIEgWwH27Nx40b//dPWkBLRIPq6BE0EgrQrpH0hg9ozz/bQPpMAjdegdI6AEwSb8ct4/bCc341pWIffj98xCO8f3Y74+ehjhMchuOMxvxEplHLGL6NENywj6GWd8Ld85hDYhWBRRESmFwWLIiKTUDSIIVBgGtKpPiYoI/Ak2CARgAw3H00jLScRBFECxDSk6OPRnguPCTaYDyVa0cRyPkOoMkngRClZeG+Ck2iiiifVN+k0J3SeQ2IZQWQU3wWBN1VjQ4czBIY7duzwVWh5LlqlmHm+Q36LaOksgS/zLCcYpFSQbQnfP49Zzt+E34F1Q4le+I1YxrphGfNh/ejf8bkpQeU9o6W90Xmm0eXDdeQSEo/B65JYh+8+fhkpLON5HjMPHvM7ht9SRESmHwWLIiLTXAhICFTONEWraTKNzjMdaXl0GuZ5PQIOSqdC4jEpBErhffkb3nu4SxZ/R/BDYBjaBpJCSSbBEUEbrxFK6wg8CQxJoR1i2Kbh3iMES2E7eT2mrMvfEOghfM9sewi0KJUj0OPv2b7RloXl0RS+E54L70+Kbk/8Y+Yp7WMav+2ksG0iIiKjUbAoIiInjUtGCPiipW/RNNry+CmJIJAAi2n8/GjLossJ1uIvZwRJBHUEkgRIvFcICAnmhhOCQBIlhQSaiC4LpXI8JpgjKOPvQFDGPOsRqILX4DHPEcCOtiws53WZhnX4LMyLiIhMJAWLIiIjGBg8Zm09A9bU1W/tbsrjvoFj1u+mgzp1njECNkr4qK7Z0UHnK7HOftx8R2x+5OWxZbHlPb09dmyg30WzLggcHBia8nigz88nuekxqkqmpLkoMjWW3OOkFJfc1M8n+0DQp6xsy83LtdKSUr+tGZlDQSKle1Tv9MsyMn0JJu36QgCZyXrub7P866gd35lJsvSUJMtMS7Ysl3IzUqw0J81SkpPcMyIiMhEULIqIxOGsOOD+19o9YI/ubbHf7miyZw61W5t7XNvWa80ueOzpH75kSs4SAsO+LrPezqHU51JHk1l73dDy1lqz9ByzvFlmOS4AzC4wS3MBXroL6NJcSnfzTGXSICCszE+3RaVZttSlC+bm2s1ry6wgM9UFkXRmNLSeiIiMHwWLIiIRlBxWt/TYD547ar/b2WS7jnZZU2e/dfUN+gCy3z3PVGfOycb9IPwovmTRJabHBtw8pY1E/25KCWIKJYqUJpJctMEyn2LzMqmkpVC6mGzpqUm+dLEoK81ev6rE3nbBLDu3cqiKsIiIjB8FiyIiMQSFj+9vdYFivT25v80ONndbR89QkCgiZx8x/cLiLLtwXp5dd06xvWZFkZXnDbUXFRGRsadgUUTEoWrpfTub7fsb6+w325us1T3WyVFkcspJT/EB41vWlvmqqYVZqZaarHqpIiJjTXVuRGRGIyCkw5oNh9rtf54+Yj/b3GAtChRFJrWO3gF7ZG+LfeXRart3e9PQMauDVkRkzClYFJEZbdAFii3d/faNJ2rsoT0t1q2Oa0SmBNoXb6nttE/8eJdtq+uy3gEduyIiY03BoojMaM0uUPz+xnpfskhVVBGZOvzNHnfcfvvJWh84iojI2FKwKCIzFtVPj7T22V0b6+xwc49/LCJTB0csJYz3bG+052s6fPVUEREZOwoWRWTGovfT56rbbeNhMpmqwiYyFREwHmjqsa21HXa4pXdooYiIjAkFiyIyY1W39vhOMjp7B2xQvWNMOgyJsKYqxy6en2/nz8617PSxv2TNLcyw9XPzbN2cXKvIT/fj+snUtLO+y/Yc7Yo9EhGRsaBgUURmrMaOftt6pNMFirEFMqncuKbUvvO2FfbAR8+zu9690haXZMWeGTvvv7TSfvn+c+0nt632wzAUZaXGnpGpZl9Ttx1s7ok9EhGRsaBgUURmLHpB3dfYbcc0UIbIlMexfEDBoojImFKwKCIzVu8APSkODDV6EpEpra17wNp71MGNiMhYUrAoIjNW38Cgb6+oWHHmemJ/m33ziRq7/cla21TdYV196uhoqqI34wHVKRcRGVMKFkVkxmL8fQ3CP7P9cmuD/dUv99rf/Xqf/X5Xs7WpZEpEROQ4BYsiIjKlJSWZpSQnWapL9GYaTSzjuZH6OE1xfxzWS+aFYpjj76LPucmI7xH9WxERkeki6ZgTmxcRmVFuf+qI3XrnttgjmWw+fHmVffDSSltRnu3H0bvpW1v8wOvxLpqXZ69cXmQXz8u3irx0y0h1waEL3mi/dqC52zYd7rBvPVVrta2JY/B95Ioqe/N5Zb796u1unbtfaLTGzn5bUpplb1hdYu++sMJq23rtm4/XWmffgN28tswWFGVabmaKDzTpJGl7Xafdv6v5+N/K2fOxK2fbF29cEnskIiJnSiWLIiIyZb1zfbn90cvm+OlViwtsdWWOzS7IsKqCdFtZkW2vWlZk77yw3P7x2gX2Sjefm54S+8shjOW4sjzHznEBaWlOmi8lRGZaslXmD43zeK57zXe51yCw5DWWlGVZlXtuYUmmrZuTZ9evLLGPuyDlLWtn2YLiTP/3IiIi04GCRRERmXLyMlJ84EaQeO05xVbpgj4GZf/vx2rsvx6ttv96pNp+tOmo7W/qsaWlWXazC+TefsEsu2BuXuwVTl6ue6+L5+fZ/KJMe3x/q33ziVr7yqM1fvr4vlbLdoHlxfPz7S3nl9lSF0iKiIhMFwoWRURkypnlgsOPXzXbV0FNT0m2TTUd9vXHa+yPf7LLPvXrfT7933v323eerrVdR7vcOkm+BPB1q0qs4BQH3icw7Rs8Zo/sbbX/d98B+9tf7bVP373P/o97D95zS22nX+9CF4guLsmyzFRdWkVEZHrQFU1ERKYUKooSwF26IN9y3HRnfad9b0OdL1WM2tPQbXc+U2f/+chhq2vvs7LcNFs3J8cumX/qpYv0lPo/zxyxJ/e3xZaYb6/45IFW+/mWo/4xJZCLSzNtblGGfywiIjLVKVgUEZEpZVFplq96WpiV6nssfdwFcFQPHU5TZ78LJOt96SJjKM4pzLTLFhTEnj15Tx1os6cOvhgoBodaev1YjQFtIEkiIiLTgYJFERGZUopckLikNNN3RtM/cMyXLBIMDoeB2o929Nnh5h7fOyp/S8c0p6K1e8Aa3Gvw9/F6+wdfMjYj1V1JIiIi04GCRRERmVJy0lOsLHeo9I4A7mhHvw/oRtPc3W/dLrBLT03yVVhPRUfvgB9aQ0REZKZRsCgiIhOuODvN1s7O9Z3CMAzFqXQKk5qSZFlpQ+sTAFJ6eCIDbp3THVW4zwWKJ/EWIiIi046CRRERmXDXLCu0n7x3lf3+o2v9GIazC0fqFCaxSmdX71C1UFCtlMDxRBU/M1wwmsIVzwV9pxs0ioiIzDQKFkVEZMIxOH5lQYYf/L4wO3XYkkWqm+ZkJC5vd8FibVsvcZ8PAhkI/0Q9kDLAPtVPaV9Y3dobWyoiIiKjUbAoIiITjqqj3X2DlpxktrI8xw9rEY+hKEis29jZd7y66cGmHntwd4t1uMDvmPu3bk6uS8MPh0GASCkmndpku+CzxgWKGw4l9moqIiIiiRQsiojIhKNjmprWHj+/qjzbLluQb0vLslzwONQBzSuXFdl5VTmWn5Hqg8rNtR1+6As0dfXb1tpO21Hf6Z9b49ZjKA0G6I8qzk61S9zrvvfiSptbmOnbHu6o77LH9g0/zIaIiIi8lIJFERGZcFQj3XCo3Xr6B317xRtWl9ot589ywV2evWJJkf3pK+bYVYsLLS0lyY64dX+7o9lau/tjfz00IP7PtzTY4ZZe3zMqweIHL6uy9XPzXJCZ69M1LuB8x/pyu/HcUstNT7btdZ0+UHzhSGfsVURERGQ0ChZFRGTCPV/TYd98otZXC8WF8/PsM9cusEc+fr799H2r7LUrin3nNQ2dffbEgTb7/sY6a+x8MVjk7z599367Z3ujHXHzcwoy7NaLKuyBj661n962yqev3rzM3umCRdo1Nri/5f3+d0Nd7BVERETkRBQsiojIhOvsHfRtB9/3vR32o01Hbc/RruPVTFHrAsAHdjfb5+8/6ILCfSMOXfH/7jton3LP/+C5ejvQ1O2Xleen+85zaA+57Uin3fVsnb3rf7bZ91ygSImkiIiInJykY05sXkRkRrn9qSN2653bYo9kotE+MTs92c6tzPFjLdKZTZof32KoTSPDY+xp6LbdDV1+2Uj42wUlmW6a4XtZ5SWS3Gv3Dgz6wfrr2nptU02Hf814F8zJsxXlWcaY+89Xd7j3Gwpai7JTfRvKNZW5/u+ePNDqt2U4pTlp9sZzS/08f0+Aqh5Xz46PXTnbvnjjktgjERE5UwoWRWTGUrAoMr0oWBQRGVuqhioiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIiIiIiCRQsCgiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIiIiIiCRQsCgiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIiIiIiCRQsCgiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIiIiIiCRQsCgiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIiIiIiCRQsCgiIiIiIiIJFCyKiIiIiIhIAgWLIiIiIiIikkDBooiIjIlZuWl245pS+4+bltpnrl1gswsyYs9MXwWZqXbFogL7ys3L7PKFBf6xiIjIdKFgUURExsQ55Tn2+lUl9vYLZtkHLq2ySxbkW3H29A6estOTbcWsbPvAZZVummVZ7rGIiMh0kXTMic2LiMwotz91xG69c1vskZypT1w1x958XqkVuQBxcWmWfcd9v197rMaePtgWW2NIZX66leakWVpKkn/c3jNg6SnJ1tU/aDUtPdbZN+hfo8ytk5uRYlykBgaPWe/AMeOK1dLVb00uUXKZ6mKzDPc/Xovn69p6rbGz38py017yHgODZoeae6yhs88/xhz394VZqZaemmTu5a3LvW+6W58p6zW51wHBIEFhFNvMNlTlZ/jP/Devnm+f+c1+u3tbo22r67S27gEfKM/KS/eviX73JrzmEbeNbGtJdppVFaT7+ay0odevb++zw+47kNPzsStn2xdvXBJ7JCIiZ0rBoojMWAoWx9a/vGGRXbWowAdLlywosNbufvvsbw/Yjzcdja0x5NOvXWAfuqzSZuWm+8cP7mmxeUUZtqWmwz7163228XC73XJ+mX3UZfwvd69DkNXiXuugC/a6XUD5s80N9v2N9favNyy24pxUW1qWZeXutQ65IOtz9x+0722ss49eMds+cOmL78Hf/9GPd/nfPPic+/ub1pTa/KJM6+gdsOdrO2xeYYZtqu7wgS6vg6f+ZJ1dMCfPzwcP7m6xHzxXZwuKM+1PXzHXCAe5mN6/s9kFyNX2u13N9tbzZ9kn3XMEpahr7/Xb/S9uG/kst15Ybp+5bqHf7tUVOX6drz5abZ/82R4/L6dOwaKIyNhSfRkRETlj77ig3NbOzrXnXcD35Ueq7bG9LVael+5Lz6L+7tXz7bpziu2e7U128Rc2+NQ/MOhLAUEJHq/z/kurbGDgmP3dr/fam7652e7Z1uhLGsPr+bI6978lpVm28VC7ffT/22mv/9pmu8sFY395zTy70gWt/7uh7vh7EGC+c325/fHL5vjt+jMXxLEdv3PB3Zu/vcXefec2/5p5GSnHSwIJQu/98Brr6Bm0T7hAk9e5ya37kAtul83Kssr8DB80fvbeAz5Q/L/37HfB7l7b3dDt3+tPXz7H7nWf8w/u2Gpv++4L9usXGu0tLggmEF5Ukum2P8mXihJMvu+u7fay/3jW/v3Bw/69RUREJgMFiyIicsYuXZDvq1LucYHS5ppO+9rjNb6q5iXz8+zCeS+Wyl22MN/aewfs97ua7akDbT497ALLmtahqpd0EPOmc0utIi/NlzgS/D3gArIvP1JjB5p6fCljvMf3tdpPXTBIiSbB3qXz84+XDob3+Mqj1dbRO2hXLS6wa5YV2mtdoEh100fde9+3o9kHjf9830Gr7+izZBfEgeqs//NMnX3+9wftJ88fPf5aj+9vdeuYDyqbu/ptX2O3X58pgSLbcMVCSkTNfralwQeMBLsErJ1uG65ZVuSr6aLPBcqULD57uMM2uKCXEkcREZHJQsGiiIicNtoEVuSl24rybDvS1mdbj3T6Kp2P7G11wV23rarIsYtcsBjWo1SPoGpLbWfsFQj22o4HSTnpKb5jnJ7+Yy7o7LBdR7t80PmIC+oONHdbl3vtKJ7b616Pdn657m8vd8FoeX66by+4fm6evf+SSp+o5kn7xDmFGXbxvHxfIkkbRv6WKqoEfT92AWGd+wwDsdYZDS5w/PaTtfZzF/Cx3ZSevmVtma8ySxvLkRRlp/n32e9ee2tthw86SZS6bnGPFxZnHi9JpVrt7qPd7vO6yFJERGSSUbAoIiKnjeCO0kICKIJEmsGfPzvXJwKh0tw0W1aW7QNFqnVmpib7Tl4aIx3N1Lb1+qAPyclJvmSud2DQ+gZeWopISWBfXMlit1sWShupwnqBCxB5jysXF9iHr6h6ScrLTPGBKp3l5Lh123v6E4K0rr6B4++bmZZsK10QfJ77LLddXGF/9cp5dutFFb7UMMdt40jS3GfIcMFkq/tMg5GX5zPx/ilJSZZK0aTDtjS6oJSpiIjIZKNgUURETltRVqovcSvJSfPTH713tT39pxf49IbVpb4Ej2qor1lRHPuL8Ufg+em799v6zz+TkG65fat95+kXO7kZzeKSLP95HvvE+famNaX2o0319uqvbLLP/f6g79FURERkulOwKCIip412iufNzvEd2nzw+zvski9sOJ4u/rcN9l0XmJXkpNraObn2XHW7dfYO2Ky8NF/SGMwvHhrCApSwUfJI6WC6S1FUMx2t+idDUFCdlaEw6ECGoTUosItPlES2dg+490yz7LSXlhDmZqT6toilbpvPrczx20YPq2//7jb7yqM17u+Hhu8YDSWIDANCVdjo5vKZzqnI9u/PtoqIiEx2ChZFZMYiFiEDL6eHgOz6lSU+8Pvx8w32i60NxzuBCYl2h/0DZivKsnyJH8Hc0tIsX100uGZpkXutoQ5fGG6DXkMJChmGg3aHlF7SVpAeSKk+OhL+9m73twebevzfvnp5UewZ86/DMBakVS5g2+QC1wVu+1e6+Vm5aUM9pF491wWY6b6DmzT3/ozxyP5BVVXaH9IustgFmGwL2zQSSh131nf6ardXuO2oyk/34ylSNZa2k5vcd1LtXktERGSyS/m0E5sXEZlRttV1+SEcaPcmp46A710XlvsB9Bk7cGd9V+yZFzHUBT1/krYf6bK23gGbV5Tp2zjS3nHdnDy72gWLjHVIIPnbHc1+vEM6zKEKK+MYznfpjeeW2JzCTF9qR0+nW2s7fdVWgjZ6U/VB6eAxq2vv86Wd55Tn2EIXgBLw8R6vX13iX5OSze3ud6fX1jWVuVbstq/SBXOUjlLVtDIvww639vhSUILOdXNyfdtF2l4uLcu2NVW5PrCc67af3lm3u89MMHndymIXDA/6Usc2FxTToQ3bz/Aaswsz/DiNVywq9Nt45zN19syhNv+dMMTHrqPd9ui+Vl/aKaePdqB0jvTaCazyLCIy3SlYFJEZi+CGIRzoDVNOHaWBBE8v1HXab7Y1WlNX4vfY6QLxjNQkF1xl+mqk33qy1ly8Z+e5v7tqSYFvz1jb2us7taFE7pdbG22H+13oLIdqpOtcQErpH5U2e11Q1t13zF440mnPumDuHBf8UZ3zMRdo0Wtq8NTBNst27xV9D16LoSvuePqIC9Tafc+kVH2l852LXYBB4LbTvUalCwRrWnrda7bZkwfa/NiLKyty7HwXNPI6dJLz93fv98EtQeELRzpshws+l83K9t9Htgss2Rb2qwYXMK53f0OpJoEqQSVDivzqhUard0HtPBdE0mvqFvd5GDaDzyynjxsDly1kaJQXS5RFROTMJB2j6zoRkRnoN9ua/KDvz7jggrZscmoYjjDZhVN8dbTlGwkdfya5lVmF9cLj4Oa1ZfbRK6psmwua3n/XDv96PJ3EvxdXs++/e6UfmP+uZ+vtPx8+fHw8xGP8i3v7+PcAl7vo7+zXIRyMrcZrPPkn62xXfaf992M1fuzFhNc5/hmGPjfvzX9+W9x/vEbYHhbx/YTXZ73B2HMIz7O+9r8zR2n1H14+2/7imrmxJSIicqbUWEdEZqzyvDRf6hMfVMjJIehhTMLRAkUQCNFxDet96w+W2x1vP8duXFPql5Eunpfnq6RSdbOqIMP+5Q2L7N4Pn+ernvI8Hc7Q2QzjEx7t7Ds+CD6vRxru7cN7RhPLQFvLH966ygWfq+zqZYX+OcY9/JOXz7by3DR7+mC7S21+Xf7mJa/D+7nl4XPz3uHx0Hu8uD1Mw/Ljfxt7DuF595SMgeWzqLqcGXskIiJjQSWLIjJjURXw3h1Nduud2xLG9JPx8dnrF/pqglQppSoo1s7O9e0Of/hsvW2r67TLFhTYp1873w+UT9tCBvQnmKO9H9VU793e5DubOV30Ukp7x7+8Zp6v+kq1UaowMjbkD9w2/PC5ett6pDO2tkwVf/ryOXbbJZW+erKIiIwNtVkUkRkrI5VKgEn2+10tvqdOghEZXwxU39M/VDqXkpzk03YXIBKg0UaQ5wje6A2Vqp2ZaSm+5JcB+X++ucH/VmcSKILXokOcstw0/9psA/cKDrf02hcfPPyS9o8y+VExgM6E3rpull0yP3/U4VVEROTUqGRRRGY0Spa+/Ei13fHUEd+7JdUCRWRqIFBkeJMPXVZl77m4wldXFhGRsaPbbyIyo1G98Y+umuN79czJ0ClRZCqhFJGebilVXFY2NFaniIiMHeWMRGRGS05OsvzMFPvYVVV2+cKC2FIRmexoy7q4JNM+cdVsW1CUoeqnIiLjQGdWEZnR6AeVNmvr5+TZey6qsJvOK/Pj76l/VJHJix5yqQ3wjvXl9rpVJVaYTfvT2JMiIjJm1GZRRCSGQeWfPtBmP3r+qN2/s9n2N3Vbd99g7FkROZsIBik9rMxPt5Xl2fbK5UV2/cpiW1am3k9FRMaLgkURkTh17X32lUeq/bAah5t7rL13wDp6BqxTgaPIhKJH3PTUJMtKTbbczBSryEv31cXfsKrE1s3NtYLM1NiaIiIyHhQsiogMg8HVq1t67amDbfbg7hZ7eE+LbTjUpgHURSZQXkaKLSjOtNWVOXbRvDy7anGhLZ+VZdl+SJXYSiIiMm4ULIqIjICB+ilVbOvut7YeShcHbWiEQJkqtm/fbrfeeqtxqfvLv/xLe+Mb3xh7RqYC2hMzHmp2WrLluMAx3yXG3nSLRURkAihYFBGRaWvDhg22fv16Hyx+5StfsQ9+8IOxZ0RERORE1BuqiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSIOmYE5sXEZGIQXd67OwdtPaeAevqG7QB97h/8JgNuKQz59TwwtbNdssbr/e/16c+81m7+a1vjz0jU0FaSpKlpyRbemqSZaWlWEFmiqUkJ8WeFRGR8aZgUURkBASJj+1rtd/uaLJnDrZbW0+/1bb1WnNXv/X069Q5FRzraLTezb/x86kLL7SUWUv8vEx+hISV+em2qDTLlrp0wdxcu/m8MsvNUMAoIjJRFCyKiET0DRzzAeFPNzfYA7uabWd9l9W391p774ANDJr1uv9Rsuj+k6lgsN+sq2VoPiPHRYyZQ/MyJaSnJFlmWrIvVcxJT7ZZeel27Ypie/PaMltZnh1bS0RExouCRRGRGEoMnznYZj9xgeKje1ttT0OXtfUQJOo0KTIZJCWZLS/LtssW5ttrXdD48iWFVpabFntWRETGmoJFERGntXvAHtrTbHdtrLdfbGnwgaNOjiKTE1VRL12Qb29bN8tuWF1q+WrLKCIyLtQbqojMeJQcPl/Tbrc/dcR+8Fy9NSlQFJnUaE9MNfEvPnTYHtzT7GsA6Na3iMjYU7AoIjMagSLtEb/+eK09uLvFuvsGY8+IyGTWO3DMNtd02Ed+uMt21HdZn6qLi4iMOQWLIjKjtXQP2A+frbenD7RZY2dfbKmITAX9LmA82tFn33nqiL1wpCO2VERExoqCRRGZsShVrGvr9e0UDzb3+J5QRWTq4Iilh+Jfv9Bgz9d0WGfvwNATIiIyJhQsisiMRSc2z9d22JMH23ybJxGZemiruKeh27bWdlpNa29sqYiIjAUFiyIyY1W39NrDu1uswwWKg+odQ2RK21HXabuPdsUeiYjIWFCwKCIzVkNnn2090qkB9kWmgX1NPXawWSWLIiJjSeMsisiM9dPNDfbJn+623Q1d6nb/LPraLctsxazs2KNT97n7D7nf8mjs0cS6ZlmR/eHlVTYrN83+d2Od3f1Co68SKROvJCfNPnJFlf39axfEloiIyJlSyaKIzFg9/YN+TEUNqnh2nVeVa1csKjjtVJmfHnuliVfmApQL5+bZ5W47FhRnWnZ6SuwZmWgt7lhu61bbYxGRsaRgUURmLHo/7egdUKw4SdS399lDe1rsjqeP+KEQTjbtqO+MvYLMZP2Dx3wSEZGxo2qoIjJj3e4CjVvv3BZ7JGfLk3+8zi6cl2eP7Wu1Lz102L63oW7KBPBvPX+W/fPrF9mcogz73P0HffDKQPFydnzsytn2xRuXxB6JiMiZUsmiiIiIiIiIJFCwKCIiIiIiIglUDVVEZixVQ50cxroa6nsvrrClZVlW29prz1V3WFZasi0pzbKK/HTLdvM0a+vuG7TDLT32+P4221bXaZ29iR2jZKQm2+yCdLtqcaEtKMq03IwUS0426x84ZvUdffb0wTZbVZFjf/6KuaqGOkmoGqqIyNhSyaKIiEwrr1tZYh++vMpuu6TSbl5bZh+4tNI+dFmVvd89fp9LH7ys0j5+1Wz7yBWz7V3ry+2CObmxv3xRngsMV1dku+cr/Gt9zK3/h1e417x46DUYLoP5yxfkW1pKUuyvREREphcFiyIiMi1VFaTb1UsL7TUrii3JxXNbj3T60svnqzt8L7iUPr7n4nLfSQ1j9AUpyUm2qCTTbnHL/+KaubamKscPs/KC+/vH3d8/V91uhIevW1lsr1tVYjkusBQREZmOFCyKiMi0VJydZnMKMuz5mg572x0v2LVf3WQ3fGOz3fKdrb7K6JG2XsvLTPVjNVLCGBS4ZS9bXOgCyQrLTEu2w8299tl7D9gbvr7Z3vjNzXbdfz9vn7p7n6/imp6S7KunioiITEcKFkVEZFKg/d+nXjPfHvjYWnvwJBNVQykFHA7tEJ852GYf/P4O21HXaV19gz7VtPbanRvqbJML9hjEvSQn1ZaUZcX+yuyieXl21eICHzTStvELDxyyX73Q6IPL8Bq/do+/9WStPbSn2ZcyioiITEcKFkVEZFLIz0yxFbOy7cpFBSedFhZnWk768CV7bT0Dtruh25473G6dLsAL+gaOWXVLrx1s7rF2t05WWoqVZKfGnjVfPXW5246BwWO2x/39Ewda3brdLxnwvb69z+7b0eSCxRYfUIqIiExHChZFRGRSONrRZw/vbbH/eeaIffck04ZD7dbc1R97hZdq7R6wQy4gHAntFvsGBi3FXQnp+TQoyU7zVVh73XMEic2d/T7AjEewSTBKwCkiIjIdKVgUEZFJYWd9l3354Wp753e3nXT63w11PmgbTlt3vx8e41SH4chITfKpf9Cs0QWK/aOMMMUwGlRLFRERmY4ULIqIyLTU6wI5qqKeqqJYySJtHp862GYdvQoGRURkZlKwKCIi09KpligGLV39vmorg/mvrcr1A/mLiIjMRLoCioiIRHT1D1q3S+mpyTanMMPSRxl0PzU5yQ+vISIiMh3pCiciIhJBqSIpzQWJDOyfn5nqg8J4FXnpNrcow3LSdSkVEZHpSVc4ERGRiJ31nbatrtNSkpJsQVGmranKdUHjS0sYGa6DwfxJ2SMM3SEiIjLVKVgUERGJ+M22Jvvp80etqavfVzH99Gvm2zsumGVzC18c/P+SBXn2zvXl9orFhbElIiIi00/SMSc2LyIyo9z+1BG79c5tsUdytjz5x+vswnl5flxEhrqoa+9zS0/+0nT/rmb7+eYGP+YifvSeVXb1skLbUttp//HQYT+8xnA+f8Niu2lNqRVmp9r9O5vtTd/cEnvGbO3sXHvPRRV260Xllp6SbEfaev0A/bVuSvXURSVZNqcgw7LTk43yxuyMFPvc/QftO26f2lzTMfQiMuE+duVs++KNS2KPRETkTKlkUUREJoX8zBQ7pzzbXra4wKXCk04ry3OsyAV8Y2lvY7fd6YLMf7rvoB8+IyMt2da7gPbac4rttSuKbXZBuj1X3W6P7mu19l4Nyi8iItOTShZFZMZSyeLk8A/XLvAldafrif2t9rudzbaldqhE749eNsdWVWTboeYeu2d7kz3mArrhvP2Ccl+iSQc1W2s77d8eOBR7Zgid2hRmpdr1K0tspXu90pw0y0gdusdKSeOTB1otxa1z8bx8K3HP/XJrgz20p8W/r5wdKlkUERlbChZFZMZSsCgyvShYFBEZW6qGKiIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiIiIiIJFCwKCIiIiIiIgkULIqIiIiIiEgCBYsiIiIiIiKSQMGiiIiMidyMFFtTlWNvPq/MrltZbIVZqbFnpq/MtGRbVJJpb1lb5qc8FhERmS50VRMRkTGxqDjTbru40m5/2wr78k1LbVVFjqWnTu/LTJELiK9eWmTfe/dKe8WSwhkRIIuIyMyhYFFERMbERfPzbXVljjV09tmcwgx73apiW1KaGXtWREREppqkY05sXkRkRrn9qSN2653bYo/kTP3DtQvsVcuKrK69z9bNybVtdZ32+d8fsrtfaIytMeTdF1bY610gWZid5h8/X91us3LTbW9jt337yVrbdbTLXr6k0N50bqmtcsHn4OAxa+8dsHr3ur0Dg/bo3lZ7YFezfejyKsvPTLHZBRlW5F6rvr3XfvBsvf1uZ7O90f3t61aWuPcYKunr6Bmw/3z4sN2zvck/xocuq7IrFxdYeV66dfcN+PeflZtme452+/Xud++B/7p5qS0ty/bzwSa3zQ/sarF5RRn2h247VpRn2wtHOu2+HU32o01HbePhdrt6aaHdvLbMytxnQ7MLon/vXvMut418ltesKLL3XlRhdR19trB4KKj++ZYG++qjNX5eTt3HrpxtX7xxSeyRiIicKZUsiojIGXuZC7pWuoDpYHOPfW9jnQ+WFrgAiOAriuDppvNKLS8j1e7Z1ujTeVW5PnA6ryrHSnPSfOD0jvWzbO3sXKtr67UnDrS69VPslcsKfRA53z2flUb7yFy73gWEBIu7XYD5wO4W29/UY289f5ZbXuze7djx98hOS/bLX7/KBZBZqXbD6hK75fwy/7rPHGxzwV+HrZ+b596jyE8JICvz0+3/vGa+nesC1n0NXf51WLfSPUcgyvJ2F4Ruqe3wn40pr8Md2FctL7I/vKLKt+N8zn0Xm1zKSU+x91xcYdcsLYq9foa9zH2e1ywvdn/bab/d0Ww76rr8a4mIiEwGChZFROSMvXZFsVW54GdHfaf9xgVVlK6lJCXZilnZvvQtIECrckHYfTub7J9/d9AnShJ7B4YquRBcXeMCtkvnF/jllAZ+4YHDduczddbdd8zSU5L8ekFBZqo9eaDNvvTQYfv6YzW2v7Hb3uyCUda6w/1NeI87nj5ii0ozfec7BKHvuKDcqlyQ+aALMP/tgUP2H+597nPBWm//MUtJHnqP9JRkt+2Z7vM0udev9q/z5Ueq7ZF9LVaUnepLNdnGu93zbD0lqD/f2mDd/YO+g5+1Lpj91dZGt/2H7AsPHvLPzXevR/XcxSUvVs/tHzxmtz9V69cLpZkiIiKTgYJFERE5bcRV9ABKddGO3gHb09BtjZ39vjrp4ZYeu2JRge/4JaxHsLSjvsue2N8aewUXZLngcqcLuhBK/fpcAHXv9iZ7ZG+rNXT02bfc62090uHeY9CvF1Cyt/FQm22u6bDs9GRfKkiAR8kmyxaXZvn0yL5Wq27ptWWzslxgW2Tnz8mzPe49qU5a29rrn/vbX+31JaPHfOhntr+p22773nb7zD373fJuX1Janpvm1++LBbfDoUTyHKql1nX6oJjXPNDU44PRzbUddoF777luG9HVN2hbaztdIPzSzyUiIjIZKFgUEZHTRnXQ86tybWFJpm050mmPu6As2N3Q5Uv+CNb8erNzXUCXYrVtvS71xdZyQVljtzW7ABMp7qpUnJPq2xD2DCQGhj39L11GgBpKJfMzUn31T0r8/vpV8+3pP1n3knTTmlLLde9PtdUCt05TV791DhN8UroY78OXV9k9H1pj93x4jX3yFXN9NdKRUCKZlepev6PfBgZiCx0+0wsuMKTkMpSQEnTWtvaMGnyKiIicLQoWRUTktFESeMu6WVaWk2ZvOrfEvvbW5XavC6hIr1pW7McevGBunu/sJdUFSYRIdKs2GOlbjTgplOaZW4N//tGLq3hxD71oF21J7sXTXBBGAPgfDx22m7+99SXp+q9ttvd/f4f94Ll6vy79u734vkOiSyhJ/PbbVtivPnCuCzTL7OE9Lfa+7+3wpZyUdo6Ez0iKLyvkdalyynMvOua+i9isiIjIJKNgUURmLIYAzJjm4wCON9oYXrWowPY1dvv2eT97/qj9dnuTT1988JBtONRu8woz7JIF+X4dSgYLslJeMh5hWW6a7/wFBJGU7lE6lxrXPjHT/VYEnCMhEAtBXKObPnu43X67w21LJFHyubeh2zr7Bv17xv/+makplubeIy8zxXfY84ZVJdbaPeADTILEB/c02/a6Tt8ucSRsB7225mYkW3Lk5flMDCky4D4j64iIiEx2yiWJyIyV6nLyOenJcSU9crJKctJ8D6aUHtLu8N8fPGz/9LuDL0n372zyVS5XVWT7Noz1Loij3SId3wS0M6SzGXT1Dvr2hvzNsrIs3zkOPZlShZXqo1lufiRUSaWzm9aeflvnXpPXDea616F0k0QVUgLXSvd6BG8EvPSKes2yQh+4Uk2U6qpsEx3Z0L6SDnYe2tPitiVWnXaU7Wju6reatl6bW5jpq+BSFZfE51lVkeO+h15rilW7lbHD7xY6JxIRkbGhYFFEZiwCEgIFRYunh05cGI4iPTXZfrGl0Y+rGG9vY481uMBocUmWD9Ker+7ww2kw1AZBJukaF8DNj/WYSjvCHz5Xb209A37MRsZaXDsn14+ft8QFXqOVBFMC+OsXGu2xva0+iL1xTenx9+B1/uZV8+3jV832jxkPkWE6LpmfbxfNy7OL5ufZP1630Cry031F1IHBY9bpgs/W7n7fKQ6flb+jtPHVK4ot3wV/bAslnZSW0uaQ1+OzNXb2+QCz2AWalExeuiDfLluYb69fXeKDz9+5AHpPg4bIGGsE8JQ+i4jI2En5tBObFxGZUejt8on9bb53S1UKPHWU3N10XpnVuO/vh88d9QPNx6M6JgPuX7W40Jey0ZaQDP2Na8rso1fMtg9cWmVHO/p94HWkrdfudUHccy6gJPha74K4t19Qbm9ZW2YLXbBZ39Hrq6huq+vyPYi+xgVtRVmp9vDeFt/zacBg+eUu6CNYDO/x6hVFPpBkGA4G7X9sX6stKM6yVy8vstsuqbDXrSq1FhcYEnDQM+rjLtgjFWWl+bEc37G+3N5/SaXvRfWPf7zLLnXBH9VVCZAZjP9aty2vdK+1tCzLDjb12K/ce7X3Drptn2XvvrDCD9lR4YJlhuj47tN1dqC5x48veeUihgjptkfd9hDsyunjZsSViwt8D7wiIjI2ko7Rwl9EZAbaeKjd/uvRavvmE7W+JElOzSyXOWe8QNrfEbxRDTQewRfVOanuSdXLnUc7fdDEY9rw4bzZOb6EkqDrIz/c6UsVyfjPdevQtpHLFO/xF9fM89VFKXn878dqbFlZtq+WSpVSeliNoiRzdmH68fcANwcOuyCN9oogsCOQ5TX4/dvd9n/zrct9m8SvP15rv9/VbPMZLiMv7fjr8BnZTkoaKVGkSinL1lTm+KFBqII6VM20zwesVLkNpV20c6TnV4JienBliI2F7vUb3Lr7G3tGbQcpJ0YV4w9eVuVvLoiIyNhQsCgiMxaZdgZcf99d2zV0wQT5yBVVvq3okwdafeke/u7V8/3YirQJZExDSvKo8slg91TnpJorA/nf8fYVvvSNAey//2y9/9vTQSD6zvXlfp6xHJ+rbvdB6VWLC+xfb1hsd20cCkYJLmXq+MRVs+19l1Taahe4i4jI2FA1VBGZsSgJordKeu6kyqBKF8cfbQZfuazIlzZmpCT7Dl+uX1Vide19dt/OZj94/eWLCuzdF1X4UkR6LF07O9detrjQ5hVnugCzxR7Y3eLXP120JbzunGJfFZZeTxmf8aJ5+faOC8rtUEuv7/mUTnZkaqDJMSW4f+B+v8sW5PubCyIiMjZUsigiMxrVF2nH9r8bjtih5h6NeTfOrl9Z7NvwMfZiUNPaY/9030H7+ZaG2BKzL795qe8UJs8FcsHXH6+xH2066quJjoWf3Lbazo0rhbrhG5tf0v5RJjfGy6SKMCWK77+0wrcDFRGRsaNgUURmtP6BY344hw/ctcMe2N3s28vJ+KEHWkp0o+MlEqAzkD6d4QT0UksHMgQDAW36evqHeiodC/RoGl8IRTVXjYE4ddAxEkOq3P625f4GxGhDq4iIyKlTsCgiMxonQIKPe7Y12RceOOR74xSRyS8tJckWlWTZ+y6p8FWI6XBJwyyKiIwt3YITkRmNvCWlXIyF964Ly33nKpQ4icjkRaC4soJedMvshtWlfvxKBYoiImNPJYsiIjEMpv7o3lbf0+Yje1v8EAgMjyAiZx9VkqmaXJab7oc9edXyInujCxRXVmTH1hARkbGmYFFEJA6d3vz7g4ftNy80WnVrr3X3DcbayylwFJlIlBZS8k8Pp4zZWeoCxSsXFdiNa0rtonl5fsgTEREZPwoWRUTi0L9JV9+AHWzqsWcOtdkje1vt8f2t9tzhdvWWKjKB6OhoXlGmrSzPtnVzcu0KFyiumJVtBS5IjO8ASURExp6CRRGREVCSSO+YVE9t7ur38zphikwcShXp4ZSgsSAzxYqy0ywnPdmSFSWKiEwIBYsiIiIiIiKSQL2hioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIpJAwaKIiIiIiIgkULAoIiIiIiIiCRQsioiIiIiISAIFiyIiIiIiIhLH7P8H8EcWx9Tag+YAAAAASUVORK5CYII=" + } + }, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we come to the flow definition. The OpenFL Workflow Interface adopts the conventions set by Metaflow, that every workflow begins with `start` and concludes with the `end` task. The aggregator begins with an optionally passed in model and optimizer. The aggregator begins the flow with the `start` task, where the list of collaborators is extracted from the runtime (`self.collaborators = self.runtime.collaborators`) and is then used as the list of participants to run the task listed in `self.next`, `aggregated_model_validation`. The model, optimizer, and anything that is not explicitly excluded from the next function will be passed from the `start` function on the aggregator to the `aggregated_model_validation` task on the collaborator. Where the tasks run is determined by the placement decorator that precedes each task definition (`@aggregator` or `@collaborator`). Once each of the collaborators (defined in the runtime) complete the `aggregated_model_validation` task, they pass their current state onto the `train` task, from `train` to `local_model_validation`, and then finally to `join` at the aggregator. It is in `join` that an average is taken of the model weights, and the next round can begin.\n", + "\n", + "![image.png](attachment:image.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class KerasMNISTFlow(FLSpec):\n", + " def __init__(self, model, rounds=3, **kwargs):\n", + " super().__init__(**kwargs)\n", + " self.model = model\n", + " self.n_rounds = rounds\n", + " self.current_round = 1\n", + "\n", + " @aggregator\n", + " def start(self):\n", + " self.collaborators = self.runtime.collaborators\n", + " self.next(self.aggregated_model_validation, foreach='collaborators')\n", + "\n", + " @collaborator\n", + " def aggregated_model_validation(self):\n", + " print(f'Performing aggregated model validation for collaborator {self.input}')\n", + " self.agg_validation_score = inference(self.model, self.test_loader, self.batch_size)\n", + " print(f'{self.input} value of {self.agg_validation_score}')\n", + " self.next(self.train)\n", + "\n", + " @collaborator\n", + " def train(self):\n", + " x_train, y_train = self.train_loader\n", + " history = self.model.fit(\n", + " x_train, y_train,\n", + " batch_size=self.batch_size,\n", + " epochs=1,\n", + " verbose=1,\n", + " )\n", + " self.loss = history.history[\"loss\"][0]\n", + " self.next(self.local_model_validation)\n", + "\n", + " @collaborator\n", + " def local_model_validation(self):\n", + " self.local_validation_score = inference(self.model, self.test_loader, self.batch_size)\n", + " print(\n", + " f'Doing local model validation for collaborator {self.input}: {self.local_validation_score}')\n", + " self.next(self.join)\n", + "\n", + " @aggregator\n", + " def join(self, inputs):\n", + " self.average_loss = sum(input.loss for input in inputs) / len(inputs)\n", + " self.aggregated_model_accuracy = sum(\n", + " input.agg_validation_score for input in inputs) / len(inputs)\n", + " self.local_model_accuracy = sum(\n", + " input.local_validation_score for input in inputs) / len(inputs)\n", + " print(f'Average aggregated model validation values = {self.aggregated_model_accuracy}')\n", + " print(f'Average training loss = {self.average_loss}')\n", + " print(f'Average local model validation values = {self.local_model_accuracy}')\n", + " print(\"Taking FedAvg of models of all collaborators\")\n", + " self.model = FedAvg([input.model for input in inputs])\n", + "\n", + " self.next(self.internal_loop)\n", + "\n", + " @aggregator\n", + " def internal_loop(self):\n", + " if self.current_round == self.n_rounds:\n", + " self.next(self.end)\n", + " else:\n", + " self.current_round += 1\n", + " self.next(self.aggregated_model_validation, foreach='collaborators')\n", + "\n", + " @aggregator\n", + " def end(self):\n", + " print(f'This is the end of the flow')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define the Participants and runtime now ! Each participant has it's own set of private attributes which can be set using callback function while instantiating the participant. The callback function returns the private attributes in form of a dictionary where the key is the attribute name, and the value is the object that will be made accessible to that participant's task\n", + "\n", + "Callback function, `callable_to_initialize_collaborator_private_attributes`, segment shards of the MNIST dataset for two collaborators: `Portland`, and `Seattle`and returns the private attribute `train_loader` and `test_loader`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openfl.experimental.interface import Aggregator, Collaborator\n", + "\n", + "# Aggregator\n", + "agg = Aggregator()\n", + "\n", + "# Setup collaborators with private attributes\n", + "collaborator_names = [\"Portland\", \"Seattle\"]\n", + "def callable_to_initialize_collaborator_private_attributes(n_collaborators, index, train_dataset, test_dataset, batch_size):\n", + " from openfl.utilities.data_splitters import EqualNumPyDataSplitter\n", + " train_splitter = EqualNumPyDataSplitter()\n", + " test_splitter = EqualNumPyDataSplitter()\n", + "\n", + " X_train, y_train = train_dataset\n", + " X_test, y_test = test_dataset\n", + "\n", + " train_idx = train_splitter.split(y_train, n_collaborators)\n", + " valid_idx = test_splitter.split(y_test, n_collaborators)\n", + "\n", + " train_dataset = X_train[train_idx[index]], y_train[train_idx[index]]\n", + " test_dataset = X_test[valid_idx[index]], y_test[valid_idx[index]]\n", + "\n", + " return {\n", + " \"train_loader\": train_dataset, \"test_loader\": test_dataset,\n", + " \"batch_size\": batch_size\n", + " }\n", + "\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0, num_gpus=0.3,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " n_collaborators=len(collaborator_names), index=idx, train_dataset=(X_train, Y_train),\n", + " test_dataset=(X_test, Y_test), batch_size=32\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=agg, collaborators=collaborators, backend=\"ray\")\n", + "print(f'Local runtime collaborators = {local_runtime.collaborators}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have our flow and runtime defined, let's run the experiment! " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "flflow = KerasMNISTFlow(model, rounds=3, checkpoint=True)\n", + "flflow.runtime = local_runtime\n", + "flflow.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Congratulations!\n", + "Now that you've completed your this notebook, see some of the more advanced things you can do in our [other tutorials](broken_link), including:\n", + "\n", + "- Using the LocalRuntime Ray Backend for dedicated GPU access\n", + "- Vertical Federated Learning\n", + "- Model Watermarking\n", + "- Differential Privacy\n", + "- And More!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "keras_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.17" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb b/openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb index 0c5f259272..34e7adcb4c 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_201_Exclusive_GPUs_with_Ray.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "14821d97", "metadata": {}, @@ -10,6 +11,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bd059520", "metadata": {}, @@ -18,6 +20,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "fc8e35da", "metadata": {}, @@ -26,6 +29,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "4dbb89b6", "metadata": {}, @@ -42,6 +46,8 @@ "source": [ "!pip install git+https://github.com/intel/openfl.git\n", "!pip install -r requirements_workflow_interface.txt\n", + "!pip install torch\n", + "!pip install torchvision\n", "\n", "# Uncomment this if running in Google Colab\n", "#!pip install -r https://raw.githubusercontent.com/intel/openfl/develop/openfl-tutorials/experimental/requirements_workflow_interface.txt\n", @@ -50,6 +56,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7237eac4", "metadata": {}, @@ -138,6 +145,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "cd268911", "metadata": {}, @@ -178,13 +186,14 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "b2e45614", "metadata": { "scrolled": true }, "source": [ - "Now we come to the updated flow definition. Here we request `@collaborator(num_gpus=1)` as the placement decorator, which will require a dedicated GPU for each collaborator task. Tune this based on your use case, but because this uses Ray internally, you can also pass through a [fraction of a GPU](https://docs.ray.io/en/latest/ray-core/tasks/using-ray-with-gpus.html#fractional-gpus), which will allow more than one task to run on each GPU (i.e. `@collaborator(num_gpus=0.5)` would result in two tasks per GPU). " + "Now we come to the updated flow definition." ] }, { @@ -215,14 +224,14 @@ " self.current_round = 0\n", " self.next(self.aggregated_model_validation,foreach='collaborators',exclude=['private'])\n", "\n", - " @collaborator(num_gpus=1)\n", + " @collaborator\n", " def aggregated_model_validation(self):\n", " print(f'Performing aggregated model validation for collaborator {self.input}')\n", " self.agg_validation_score = inference(self.model,self.test_loader)\n", " print(f'{self.input} value of {self.agg_validation_score}')\n", " self.next(self.train)\n", "\n", - " @collaborator(num_gpus=1)\n", + " @collaborator\n", " def train(self):\n", " \"\"\"\n", " Train the model.\n", @@ -247,7 +256,7 @@ " self.loss = loss.item()\n", " self.next(self.local_model_validation)\n", "\n", - " @collaborator(num_gpus=1)\n", + " @collaborator\n", " def local_model_validation(self):\n", " self.local_validation_score = inference(self.model,self.test_loader)\n", " print(f'Doing local model validation for collaborator {self.input}: {self.local_validation_score}')\n", @@ -271,19 +280,18 @@ " \n", " @aggregator\n", " def end(self):\n", - " print(f'This is the end of the flow') " + " print(f'This is the end of the flow')" ] }, { + "attachments": {}, "cell_type": "markdown", - "id": "7a133f9f", + "id": "49c4afa8", "metadata": {}, "source": [ - "You'll notice in the `FederatedFlow` definition above that there were certain attributes that the flow was not initialized with, namely the `train_loader` and `test_loader` for each of the collaborators. These are **private_attributes** that are exposed only throught he runtime. Each participant has it's own set of private attributes: a dictionary where the key is the attribute name, and the value is the object that will be made accessible through that participant's task. \n", + "In this step we define entities necessary to run the flow and create a function which returns dataset as private attributes of collaborator. As described in [quickstart](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb) we define entities necessary for the flow.\n", "\n", - "Below, we segment shards of the MNIST dataset for **four collaborators**: Portland, Seattle, Chandler, and Portland. Each has their own slice of the dataset that's accessible via the `train_loader` or `test_loader` attribute. Note that the private attributes are flexible, and you can choose to pass in a completely different type of object to any of the collaborators or aggregator (with an arbitrary name). These private attributes will always be filtered out of the current state when transfering from collaborator to aggregator, or vice versa. \n", - "\n", - "The LocalRuntime is now initialized **without** the backend argument. Now the LocalRuntime will default to `backend='ray'`, which allows passing through the `num_gpus` argument in the placement decorator" + "To request GPU(s) with ray-backend, we specify `num_gpus=0.3` as the argument while instantiating Aggregator and Collaborator, this will reserve 0.3 GPU for each of the 2 collaborators and the aggregator and therefore require a dedicated GPU for the experiment. Tune this based on your use case, for example `num_gpus=0.4` for an experiment with 4 collaborators and the aggregator will require 2 dedicated GPUs. **NOTE:** Collaborator cannot span over multiple GPUs, for example `num_gpus=0.4` with 5 collaborators will require 3 dedicated GPUs. In this case collaborator 1 and 2 use GPU#1, collaborator 3 and 4 use GPU#2, and collaborator 5 uses GPU#3." ] }, { @@ -293,39 +301,52 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup participants\n", - "aggregator = Aggregator()\n", - "aggregator.private_attributes = {}\n", + "# Setup Aggregator private attributes via callable function\n", + "aggregator = Aggregator(num_gpus=0.3)\n", + "\n", + "collaborator_names = ['Portland', 'Seattle']\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index, n_collaborators,\n", + " train_dataset, test_dataset, batch_size_train):\n", + " local_train = deepcopy(train_dataset)\n", + " local_test = deepcopy(test_dataset)\n", + " local_train.data = train_dataset.data[index::n_collaborators]\n", + " local_train.targets = train_dataset.targets[index::n_collaborators]\n", + " local_test.data = test_dataset.data[index::n_collaborators]\n", + " local_test.targets = test_dataset.targets[index::n_collaborators]\n", "\n", - "# Setup collaborators with private attributes\n", - "collaborator_names = ['Portland', 'Seattle', 'Chandler','Bangalore']\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", - "for idx, collaborator in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx::len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx::len(collaborators)]\n", - " local_test.data = mnist_test.data[idx::len(collaborators)]\n", - " local_test.targets = mnist_test.targets[idx::len(collaborators)]\n", - " collaborator.private_attributes = {\n", - " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", - " 'test_loader': torch.utils.data.DataLoader(local_test,batch_size=batch_size_train, shuffle=True)\n", + " return {\n", + " 'train_loader': torch.utils.data.DataLoader(local_train,batch_size=batch_size_train, shuffle=True),\n", + " 'test_loader': torch.utils.data.DataLoader(local_test, batch_size=batch_size_train, shuffle=True)\n", " }\n", "\n", + "# Setup collaborators private attributes via callable function\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0, num_gpus=0.3,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, n_collaborators=len(collaborator_names),\n", + " train_dataset=mnist_train, test_dataset=mnist_test, batch_size_train=batch_size_train\n", + " )\n", + " )\n", + " \n", "# The following is equivalent to\n", "# local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, **backend='ray'**)\n", - "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators)\n", + "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend='ray')\n", "print(f'Local runtime collaborators = {local_runtime.collaborators}')" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "0525eaa9", "metadata": {}, "source": [ "Now that we have our flow and runtime defined, let's run the experiment! \n", "\n", - "(If you run this example on Google Colab with the GPU Runtime, you should see one task executing at a time.)" + "(If you run this example on Google Colab with the GPU Runtime, you should see two task executing at a time.)" ] }, { @@ -338,12 +359,13 @@ "model = None\n", "best_model = None\n", "optimizer = None\n", - "flflow = CollaboratorGPUFlow(model,optimizer,checkpoint=True)\n", + "flflow = CollaboratorGPUFlow(model, optimizer, checkpoint=True)\n", "flflow.runtime = local_runtime\n", "flflow.run()" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "10616d60", "metadata": {}, @@ -364,6 +386,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8e084b41", "metadata": {}, @@ -381,16 +404,6 @@ "run_id = flflow._run_id" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "composed-burst", - "metadata": {}, - "outputs": [], - "source": [ - "import metaflow" - ] - }, { "cell_type": "code", "execution_count": null, @@ -413,6 +426,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "f8f7d05f", "metadata": {}, @@ -441,6 +455,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a206b36c", "metadata": {}, @@ -459,6 +474,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bf4ec317", "metadata": {}, @@ -497,11 +513,12 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "0c5522b7", "metadata": {}, "source": [ - "Now we see **12** steps: **4** collaborators each performed **3** rounds of model training " + "Now we see **6** steps: **2** collaborators each performed **3** rounds of model training " ] }, { @@ -511,7 +528,7 @@ "metadata": {}, "outputs": [], "source": [ - "t = Task(f'FederatedFlow/{run_id}/train/9')" + "t = Task(f'CollaboratorGPUFlow/{run_id}/train/11')" ] }, { @@ -525,6 +542,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "efd5da76", "metadata": {}, @@ -553,6 +571,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "3e92fab0", "metadata": {}, @@ -571,6 +590,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "ced6e90e", "metadata": {}, @@ -589,6 +609,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8b9f8d25", "metadata": {}, @@ -601,21 +622,13 @@ "- Differential Privacy\n", "- And More!" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "34fcbaa6", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { "kernelspec": { - "display_name": "workflow-interface-py38", + "display_name": "runtime-env", "language": "python", - "name": "workflow-interface-py38" + "name": "python3" }, "language_info": { "codemirror_mode": { @@ -627,12 +640,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "vscode": { - "interpreter": { - "hash": "a9b3ea793f0a9a343a81a73b472831cd604f7c2c0cb7677aa4f9120271015e80" - } + "version": "3.8.0" } }, "nbformat": 4, diff --git a/openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb b/openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb index 72d9eacd49..b6c1706162 100644 --- a/openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb +++ b/openfl-tutorials/experimental/Workflow_Interface_301_MNIST_Watermarking.ipynb @@ -1,6 +1,7 @@ { "cells": [ { + "attachments": {}, "cell_type": "markdown", "id": "dc13070c", "metadata": {}, @@ -11,6 +12,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "8f28c451", "metadata": {}, @@ -27,6 +29,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "a4394089", "metadata": {}, @@ -35,6 +38,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "857f9995", "metadata": {}, @@ -51,8 +55,9 @@ "source": [ "!pip install git+https://github.com/intel/openfl.git\n", "!pip install -r requirements_workflow_interface.txt\n", - "!pip install matplotlib\n", + "!pip install torch\n", "!pip install torchvision\n", + "!pip install matplotlib\n", "!pip install git+https://github.com/pyviz-topics/imagen.git@master\n", "\n", "\n", @@ -63,6 +68,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "7bd566df", "metadata": {}, @@ -195,6 +201,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "f0c55175", "metadata": {}, @@ -375,6 +382,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "d82d34fd", "metadata": {}, @@ -430,7 +438,7 @@ "\n", "Notice that both the PRE-TRAIN and RE-TRAIN tasks are defined as Aggregator processing tasks\n", "\n", - "![image.png](attachment:image.png)\\\n", + "![image.png](attachment:image.png)\n", "\n", "
Workflow for Watermarking" ] @@ -479,6 +487,7 @@ " self.model.parameters(), lr=watermark_retrain_learning_rate\n", " )\n", " self.round_number = round_number\n", + " self.watermark_pretraining_completed = False\n", "\n", " @aggregator\n", " def start(self):\n", @@ -668,6 +677,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "c6da2c42", "metadata": {}, @@ -676,7 +686,7 @@ "\n", "- Collaborator attributes are created in the same manner as described in [quickstart](https://github.com/psfoley/openfl/blob/experimental-workflow-interface/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb)\n", "\n", - "- `watermark_data_loader` is created as a **private attribute** of the Aggregator and it is exposed only via the runtime. This property enables the Watermark dataset to be hidden from the collaborators as Aggregator private attributes are filtered before the state is transferred to Collaborators (in the same manner as Collaborator private attributes are hidden from Aggregator)\n", + "- `watermark_data_loader` is created as a **private attribute** of the Aggregator which is set by `callable_to_initialize_aggregator_private_attributes` callable function. It is exposed only via the runtime. This property enables the Watermark dataset to be hidden from the collaborators as Aggregator private attributes are filtered before the state is transferred to Collaborators (in the same manner as Collaborator private attributes are hidden from Aggregator)\n", "\n", "Lets define these attributes along with some other parameters (seed, batch-sizes, optimizer parameters) and create the LocalRuntime" ] @@ -708,15 +718,18 @@ "watermark_pretrain_learning_rate = 1e-1\n", "watermark_pretrain_momentum = 5e-1\n", "watermark_pretrain_weight_decay = 5e-05\n", - "watermark_retrain_learning_rate = 5e-3\n" + "watermark_retrain_learning_rate = 5e-3" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "3d7ce52f", "metadata": {}, "source": [ - "## Setup Federation" + "Private attributes can be set using callback function while instantiating the participant\n", + "\n", + "Aggregator callable function `callable_to_initialize_aggregator_private_attributes` returns `watermark_data_loader`, `pretrain_epochs`, `retrain_epochs`, `watermark_acc_threshold`, and `watermark_pretraining_completed`. Collaborator callable function `callable_to_initialize_aggregator_private_attributes` returns `train_loader` and `test_loader` of the collaborator." ] }, { @@ -726,19 +739,24 @@ "metadata": {}, "outputs": [], "source": [ - "# Setup Aggregator with private attributes\n", - "aggregator = Aggregator()\n", - "aggregator.private_attributes = {\n", - " \"watermark_data_loader\": torch.utils.data.DataLoader(\n", - " watermark_data, batch_size=batch_size_watermark, shuffle=True\n", - " ),\n", - " \"pretrain_epochs\": 25,\n", - " \"retrain_epochs\": 25,\n", - " \"watermark_acc_threshold\": 0.98,\n", - " \"watermark_pretraining_completed\": False,\n", - "}\n", + "def callable_to_initialize_aggregator_private_attributes(watermark_data, batch_size):\n", + " return {\n", + " \"watermark_data_loader\": torch.utils.data.DataLoader(\n", + " watermark_data, batch_size=batch_size, shuffle=True\n", + " ),\n", + " \"pretrain_epochs\": 25,\n", + " \"retrain_epochs\": 25,\n", + " \"watermark_acc_threshold\": 0.98,\n", + " }\n", + "\n", + "# Setup Aggregator private attributes via callable function\n", + "aggregator = Aggregator(\n", + " name=\"agg\",\n", + " private_attributes_callable=callable_to_initialize_aggregator_private_attributes,\n", + " watermark_data=watermark_data,\n", + " batch_size=batch_size_watermark,\n", + " )\n", "\n", - "# Setup Collaborators with private attributes\n", "collaborator_names = [\n", " \"Portland\",\n", " \"Seattle\",\n", @@ -746,30 +764,38 @@ " \"Bangalore\",\n", " \"New Delhi\",\n", "]\n", - "print(f\"Creating collaborators {collaborator_names}\")\n", - "collaborators = [Collaborator(name=name) for name in collaborator_names]\n", - "\n", - "for idx, collaborator in enumerate(collaborators):\n", - " local_train = deepcopy(mnist_train)\n", - " local_test = deepcopy(mnist_test)\n", - " local_train.data = mnist_train.data[idx :: len(collaborators)]\n", - " local_train.targets = mnist_train.targets[idx :: len(collaborators)]\n", - " local_test.data = mnist_test.data[idx :: len(collaborators)]\n", - " local_test.targets = mnist_test.targets[idx :: len(collaborators)]\n", - " collaborator.private_attributes = {\n", - " \"train_loader\": torch.utils.data.DataLoader(\n", - " local_train, batch_size=batch_size_train, shuffle=True\n", - " ),\n", - " \"test_loader\": torch.utils.data.DataLoader(\n", - " local_test, batch_size=batch_size_train, shuffle=True\n", - " ),\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index, n_collaborators, batch_size, train_dataset, test_dataset):\n", + " train = deepcopy(train_dataset)\n", + " test = deepcopy(test_dataset)\n", + " train.data = train_dataset.data[index::n_collaborators]\n", + " train.targets = train_dataset.targets[index::n_collaborators]\n", + " test.data = test_dataset.data[index::n_collaborators]\n", + " test.targets = test_dataset.targets[index::n_collaborators]\n", + "\n", + " return {\n", + " \"train_loader\": torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True),\n", + " \"test_loader\": torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=True),\n", " }\n", "\n", - "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators)\n", + "# Setup Collaborators private attributes via callable function\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0, num_gpus=0,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx, n_collaborators=len(collaborator_names),\n", + " train_dataset=mnist_train, test_dataset=mnist_test, batch_size=64\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators, backend=\"ray\")\n", "print(f\"Local runtime collaborators = {local_runtime.collaborators}\")" ] }, { + "attachments": {}, "cell_type": "markdown", "id": "02935ccf", "metadata": {}, @@ -818,6 +844,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "bf66c1cd", "metadata": {}, @@ -836,22 +863,6 @@ "if flflow._checkpoint:\n", " InspectFlow(flflow, flflow._run_id, show_html=True)" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f60118f1", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fefef69c", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb b/openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb new file mode 100644 index 0000000000..6149abbb3b --- /dev/null +++ b/openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb @@ -0,0 +1,822 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Workflow Interface 401: Synthetic non-IID Dataset with FedProx Optimizer\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/intel/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_401_FedProx_with_Synthetic_nonIID.ipynb)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this OpenFL workflow interface tutorial, we shall learn how to implement FedProx and compare its performance with FedAvg algorithm using a Synthetic non-IID dataset. Reference: [Federated Optimization in Heterogeneous Networks](https://arxiv.org/pdf/1812.06127.pdf)." + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Getting Started" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we start by installing the necessary dependencies for the workflow interface" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/intel/openfl.git\n", + "!pip install -r requirements_workflow_interface.txt\n", + "!pip install torch\n", + "!pip install torchvision\n", + "!pip install matplotlib\n", + "!pip install seaborn\n", + "\n", + "# Uncomment following lines if running in Google Colab\n", + "# import os\n", + "# os.environ[\"USERNAME\"] = \"colab\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next we import necessary libraries, and define Synthetic non-iid dataset as described in [Federated Optimization in Heterogeneous Networks](https://arxiv.org/pdf/1812.06127.pdf)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch as pt\n", + "import torch.utils.data as data\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", + "import numpy as np\n", + "\n", + "import random\n", + "import collections\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "import warnings\n", + "warnings.filterwarnings(\"ignore\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "RANDOM_SEED = 10\n", + "batch_size = 10\n", + "\n", + "# Sets seed to reproduce the results\n", + "def set_seed(seed):\n", + " pt.manual_seed(seed)\n", + " pt.cuda.manual_seed_all(seed)\n", + " pt.use_deterministic_algorithms(True)\n", + " pt.backends.cudnn.deterministic = True\n", + " pt.backends.cudnn.benchmark = False\n", + " pt.backends.cudnn.enabled = False\n", + " np.random.seed(seed)\n", + " random.seed(seed)\n", + "\n", + "set_seed(RANDOM_SEED)\n", + "\n", + "\n", + "def one_hot(labels, classes):\n", + " return np.eye(classes)[labels]\n", + "\n", + "\n", + "def softmax(x):\n", + " ex = np.exp(x)\n", + " sum_ex = np.sum(np.exp(x))\n", + " return ex / sum_ex\n", + "\n", + "\n", + "def generate_synthetic(alpha, beta, iid, num_collaborators, num_classes):\n", + " dimension = 60\n", + " NUM_CLASS = num_classes\n", + " NUM_USER = num_collaborators\n", + "\n", + " samples_per_user = np.random.lognormal(4, 2, (NUM_USER)).astype(int) + 50\n", + " num_samples = np.sum(samples_per_user)\n", + "\n", + " X_split = [[] for _ in range(NUM_USER)]\n", + " y_split = [[] for _ in range(NUM_USER)]\n", + "\n", + " #### define some eprior ####\n", + " mean_W = np.random.normal(0, alpha, NUM_USER)\n", + " mean_b = mean_W\n", + " B = np.random.normal(0, beta, NUM_USER)\n", + " mean_x = np.zeros((NUM_USER, dimension))\n", + "\n", + " diagonal = np.zeros(dimension)\n", + " for j in range(dimension):\n", + " diagonal[j] = np.power((j + 1), -1.2)\n", + " cov_x = np.diag(diagonal)\n", + "\n", + " for i in range(NUM_USER):\n", + " if iid == 1:\n", + " mean_x[i] = np.ones(dimension) * B[i] # all zeros\n", + " else:\n", + " mean_x[i] = np.random.normal(B[i], 1, dimension)\n", + "\n", + " if iid == 1:\n", + " W_global = np.random.normal(0, 1, (dimension, NUM_CLASS))\n", + " b_global = np.random.normal(0, 1, NUM_CLASS)\n", + "\n", + " for i in range(NUM_USER):\n", + "\n", + " W = np.random.normal(mean_W[i], 1, (dimension, NUM_CLASS))\n", + " b = np.random.normal(mean_b[i], 1, NUM_CLASS)\n", + "\n", + " if iid == 1:\n", + " W = W_global\n", + " b = b_global\n", + "\n", + " xx = np.random.multivariate_normal(\n", + " mean_x[i], cov_x, samples_per_user[i])\n", + " yy = np.zeros(samples_per_user[i])\n", + "\n", + " for j in range(samples_per_user[i]):\n", + " tmp = np.dot(xx[j], W) + b\n", + " yy[j] = np.argmax(softmax(tmp))\n", + "\n", + " X_split[i] = xx.tolist()\n", + " y_split[i] = yy.tolist()\n", + "\n", + " return X_split, y_split\n", + "\n", + "\n", + "class SyntheticFederatedDataset:\n", + " def __init__(self, num_collaborators, batch_size=1, num_classes=10, **kwargs):\n", + " self.batch_size = batch_size\n", + " X, y = generate_synthetic(0.0, 0.0, 0, num_collaborators, num_classes)\n", + " X = [np.array([np.array(sample).astype(np.float32)\n", + " for sample in col]) for col in X]\n", + " y = [np.array([np.array(one_hot(int(sample), num_classes))\n", + " for sample in col]) for col in y]\n", + " self.X_train_all = np.array([col[:int(0.9 * len(col))] for col in X], dtype=np.ndarray)\n", + " self.X_valid_all = np.array([col[int(0.9 * len(col)):] for col in X], dtype=np.ndarray)\n", + " self.y_train_all = np.array([col[:int(0.9 * len(col))] for col in y], dtype=np.ndarray)\n", + " self.y_valid_all = np.array([col[int(0.9 * len(col)):] for col in y], dtype=np.ndarray)\n", + "\n", + " def split(self, index):\n", + " return {\n", + " \"train_loader\":\n", + " data.DataLoader(\n", + " data.TensorDataset(\n", + " pt.from_numpy(self.X_train_all[index]),\n", + " pt.from_numpy(self.y_train_all[index])\n", + " ), \n", + " batch_size=batch_size, shuffle=True\n", + " ),\n", + " \"test_loader\":\n", + " data.DataLoader(\n", + " data.TensorDataset(\n", + " pt.from_numpy(self.X_valid_all[index]),\n", + " pt.from_numpy(self.y_valid_all[index])\n", + " ), \n", + " batch_size=batch_size, shuffle=True\n", + " )\n", + " }" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we have defined dataset class. Let define model, optimizer, and some helper functions like we would for any other deep learning experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openfl.interface.aggregation_functions.weighted_average import weighted_average as wa\n", + "\n", + "\n", + "class Net(nn.Module):\n", + " \"\"\"\n", + " Model to train the dataset\n", + "\n", + " Args:\n", + " None\n", + " \n", + " Returns:\n", + " model: class Net object\n", + " \"\"\"\n", + " def __init__(self):\n", + " # Set RANDOM_STATE to reproduce same model\n", + " pt.set_rng_state(pt.manual_seed(RANDOM_SEED).get_state())\n", + " super(Net, self).__init__()\n", + " self.linear1 = nn.Linear(60, 100)\n", + " self.linear2 = nn.Linear(100, 10)\n", + "\n", + " def forward(self, x):\n", + " x = self.linear1(x)\n", + " x = self.linear2(x)\n", + " return x\n", + "\n", + "\n", + "def cross_entropy(output, target):\n", + " \"\"\"\n", + " cross-entropy metric\n", + "\n", + " Args:\n", + " output: model ouput,\n", + " target: target label\n", + "\n", + " Returns:\n", + " crossentropy_loss: float\n", + " \"\"\"\n", + " return F.cross_entropy(output, pt.max(target, 1)[1])\n", + "\n", + "\n", + "def compute_loss_and_acc(network, dataloader):\n", + " \"\"\"\n", + " Model test method\n", + "\n", + " Args:\n", + " network: class Net object (model)\n", + " dataloader: torch.utils.data.DataLoader\n", + "\n", + " Returns:\n", + " (accuracy,\n", + " loss,\n", + " correct,\n", + " dataloader_size)\n", + " \"\"\"\n", + " network.eval()\n", + " test_loss = 0\n", + " correct = 0\n", + " with pt.no_grad():\n", + " for data, target in dataloader:\n", + " output = network(data)\n", + " test_loss += cross_entropy(output, target).item()\n", + " tar = target.argmax(dim=1, keepdim=True)\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + " correct += pred.eq(tar).sum().cpu().numpy()\n", + " dataloader_size = len(dataloader.dataset)\n", + " test_loss /= dataloader_size\n", + " accuracy = float(correct / dataloader_size)\n", + " return accuracy, test_loss, correct\n", + "\n", + "\n", + "def weighted_average(tensors, weights):\n", + " \"\"\"\n", + " Take weighted average of models / optimizers / loss / accuracy\n", + " Incase of taking weighted average of optimizer do the following steps:\n", + " 1. Call \"_get_optimizer_state\" (openfl.federated.task.runner_pt._get_optimizer_state)\n", + " pass optimizer to it, to take optimizer state dictionary.\n", + " 2. Pass optimizer state dictionaries list to here.\n", + " 3. To set the weighted average optimizer state dictionary back to optimizer,\n", + " call \"_set_optimizer_state\" (openfl.federated.task.runner_pt._set_optimizer_state)\n", + " and pass optimizer, device, and optimizer dictionary received in step 2.\n", + "\n", + " Args:\n", + " tensors: Models state_dict list or optimizers state_dict list or loss list or accuracy list\n", + " weights: Weight for each element in the list\n", + "\n", + " Returns:\n", + " dict: Incase model list / optimizer list OR\n", + " float: Incase of loss list or accuracy list\n", + " \"\"\"\n", + " # Check the type of first element of tensors list\n", + " if type(tensors[0]) in (dict, collections.OrderedDict):\n", + " optimizer = False\n", + " # If __opt_state_needed found then optimizer state dictionary is passed\n", + " if \"__opt_state_needed\" in tensors[0]:\n", + " optimizer = True\n", + " # Remove __opt_state_needed from all state dictionary in list\n", + " [tensor.pop(\"__opt_state_needed\") for tensor in tensors]\n", + " tmp_list = []\n", + " # Take keys in order to rebuild the state dictionary taking keys back up\n", + " input_state_dict_keys = tensors[0].keys()\n", + " for tensor in tensors:\n", + " # Append values of each state dictionary in list\n", + " # If type(value) is Tensor then it needs to be detached\n", + " tmp_list.append(np.array([value.detach() if type(value) is pt.Tensor else value for value in tensor.values()], dtype=object))\n", + " # Take weighted average of list of arrays\n", + " # new_params passed is weighted average of each array in tmp_list\n", + " new_params = wa(tmp_list, weights)\n", + " new_state = {}\n", + " # Take weighted average parameters and building a dictionary\n", + " [new_state.update({k:new_params[i]}) if optimizer else new_state.update({k:pt.from_numpy(new_params[i].numpy())}) \\\n", + " for i, k in enumerate(input_state_dict_keys)]\n", + " return new_state\n", + " else:\n", + " return wa(tensors, weights)" + ] + }, + { + "attachments": { + "federated-flow-diagram.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAzMAAAJZCAYAAACds2s+AAAAAXNSR0IArs4c6QAACih0RVh0bXhmaWxlACUzQ214ZmlsZSUyMGhvc3QlM0QlMjJhcHAuZGlhZ3JhbXMubmV0JTIyJTIwbW9kaWZpZWQlM0QlMjIyMDIzLTA1LTA1VDA5JTNBMzklM0E1OS4wNTVaJTIyJTIwYWdlbnQlM0QlMjJNb3ppbGxhJTJGNS4wJTIwKFdpbmRvd3MlMjBOVCUyMDEwLjAlM0IlMjBXaW42NCUzQiUyMHg2NCklMjBBcHBsZVdlYktpdCUyRjUzNy4zNiUyMChLSFRNTCUyQyUyMGxpa2UlMjBHZWNrbyklMjBDaHJvbWUlMkYxMTMuMC4wLjAlMjBTYWZhcmklMkY1MzcuMzYlMjIlMjBldGFnJTNEJTIybjRCVEJwRzN5TXkya3JubmRGVXYlMjIlMjB2ZXJzaW9uJTNEJTIyMjEuMS43JTIyJTIwdHlwZSUzRCUyMmRldmljZSUyMiUzRSUzQ2RpYWdyYW0lMjBuYW1lJTNEJTIyUGFnZS0xJTIyJTIwaWQlM0QlMjJnMDBDN1BBcVZyaXRlTVVsdmxnWiUyMiUzRTdWeGRkNm80RlAwMVByWXJmSWslMkJVc3J0eDlqYXBYWTZmWEpSaWNnVWpZTjRxJTJGZlhUNUNnSkdCRmlvQnQlMkJsSVNra055Mkp0a244UTBKSDI2dXZITSUyQmVRQldkQnRpTUJhTmFUcmhpZ0tLZ0Q0WDVDekRuTVVRUWt6Yk0lMkJ4U0tGZFJ0JTJGNUEwa21xV2N2SFFzdXFJSSUyQlFxN3Z6T25NRVpyTjRNaW44a3pQUXg5MHNURnk2YWZPVFJzbU12b2owMDNtdmppV1B3bHp0OTBLOG0lMkJoWTAlMkJpSjRQb3p0U01DcE9NeGNTMDBFY3NTeklha3U0aDVJZFgwNVVPM2NCNWtWJTJGQ2VyJTJGMjNOMDJ6SU16UDBzRjBYeWZ0JTJCJTJGdUxRZGFvJTJGSDk3ZEo4ZUgyNWFJcWhtZCUyQm11eVE5SnEzMTE1RUxiQTh0NTZRWTlIeTRTbk84JTJCUllWQjhtR0NkdnVZcHhBTklXJTJCdDhaRmlLRUxRU0oxQ0Vaa2t2ellPYndWNFdFU2MzWXo4clZKWHJLOXRiM3pBNzRncmpqQ0xVS0tWNW91ZnV6VkdPRk94ZDNUJTJGRzhadk1FcjZzb20lMkZ6ZFZGbk56bGxvbHNIV3gyRUJld3dVRWViNUtOZFFmYUwxQlpBMzNKelJJUCUyQlROWTNQMnRwUjViSHZQVXhzeTBHemJnN2JwSXklMkYyOU5EdW9hY25Daktnd3BpYVdUQjRCd0slMkIlMkZURnhmTmlmbTZQZzdnZiUyQmt1QzhpVDkxeWUyeDQ3bzZjbkV6Z3JxU0NPVjJTeVk5aWVXUE4zJTJCQngzMFB2Y040RFlQVW1LTGZCS21CWFE5aU44VFR5RGY5V0JwJTJGekdBOERTMG5ublRSNkgzVEJZQVROQSUyQkViWjlUS0JPbnhpZXMzRXVZN2NjeFJoQ3BwU1FKSXAlMkJLSDFLdCUyQktGM0g1NmVCOGF3MCUyQjMzaDlyajlWRFQ5ZWVlcHI5V3dSbU1PdGQ4UTE3dFdBUEFHeGlMeDdBR2dLYWglMkZmbzJyRkhGUzFHaEJwcFdra2Q0VUNtUlJ6TG5FZWZSdWZHb2RpeFNPSXM0aTg2TVJZcGNQeDYxRTI4WVdsZ01reVR5JTJGQW15MGN4MGpWM3UxUTREZ1FOM1pUb0l6WWtQJTJGNFclMkJ2eWJLM2x6NmlNWUY5cUszJTJGaWVvZjZsRXlWZGlicE80WGxHcE5Va2w4TkV5VlZqUkhEOXdVeDUwWUYlMkJqcFRlQ2g1V29iM28yJTJGTXlnbEk0MkQ3cW03JTJGeW1XNWNHSEZMMUNUbWJidzVCcWFRR2I0WFM1b3lOc0FPa1dqejZjTkJTaTdFVWRqRmhhWVBrYlklMkZ5Z3p2cSUyRkhtZ0c2NGNQMVlOcDE0amklMkZoNlZ5bElmQ2RHRkk1Z0dZaVhvQjM3VXlrUUNpTEloJTJCZERkcHVnWEhSbmlPUEJtYVVGRVZHY0dybm1ZdUdNYUt6U1lNJTJCRFFQNHRMJTJCUmJMcWN6SVRZdlVGS21CVkhlVndtajBNRllFYWo1R0tJQTJwQ2dNSVpPVFltMElCV254SGxTUXFtVUVrQmdrSnh6MEdBWE9vUjJ5YU5Fb2ZHbXJQSjByMUslMkIwUWEzUm0lMkZZTSUyRnJQblVHb2xRZmFYOGJ3eGJpN3VSMFlXRG4lMkZiZlMwRyUyQk5vNGZwOWx6NEFVSVZSJTJCeGpaakNmZVFOZSUyRmkyemVMbjJ3OHl4cUtVUyUyQkxITXhSQ3cwJTJGdlJsVnZXTmpxRVBobnEzMDlHdXVqMXQwTzMxT1lNNGd5SUdBZlZTcFpWNDJvSjc2U1JTazVNekxzJTJGck5xTTdpVHluSjFjaXU2bmpHRVV1cE9PNnJPbFZxdzZLb3g0NHk4dXNreXNPT2FQaUVDdFY0UzFXaE9ja0JSNjcwdWRMWlZHaXpTbFJlMG9vWjBHSlJHQ0toWEwyZ2FKYVRrUkJnSnJvaFVGUHUzc2NocW9CYSUyQjdLZFFOZnNxNnJjbURHRWtsTXlvWnlGNnlsUXZmcGNpWnhKcDJZU2ZWaGpsZ2tjNzY4WnlvamMlMkZpMnFSJTJGTG5PMjJxZHFOUVduUjM5SVZ6czllWmp5OE95cmElMkZITklDb21WTGpSS0toUEFhdVdVT096YXU5UXVkJTJCMWRhaFpPaXJNR2VCNUtuNTRVV2VNRDFaS0NsZXU1U1pING1hRlVzdTVYJTJCVWhSOVVoeEVPdFMxaDBwVXJ0S1VqRHhZWm1kM3VTTkQ4dHNvUG5Vbk9CTEp2V1BEMGNEUmMwNXdRYUlsYUlXVFVvblJkcWlTWFhpJTJGTDU3OTFpRkNPY2JTR29wd2RrdFdGdDJWTGg3Uks3WEQ5S054MnRPbU9UaEN6JTJGOXVJWm9YS3JCbmtXNVhyJTJCWnZYc2NHTDFIclRQc2RMdFBuRHA4ck5rMzFxVDhSclpzNmlocG0lMkJnTGtTeVJTSG1sNU1zQkdiOVZEckZhUjhsNFRUVTIzOXFUeXdnNTZ6NlRQY2RGbGFNaW1nZUNSRmxGUkpNTlcyWFRFQmcyNWpwV2JCNFVXT3h2Ym9zZFhBRDR0RmxzZWZyUU1Id1JOcUJRUGFPMGtnVGhlM3ZyRmh3clkyOXYzdEJ0MHBTVWtVJTJCRllmaXNEa2ZnR0Q0WmhxWDgydzVaVTJYSGxhS1djd3olMkZjQXpuWGk1SU1jViUyQjBrJTJCTllYNk1COGZ3WnFMN2xaTTdHRk9GYlFESHlkMkJ2bUh4M2JISWt2RSUyRiUzQyUyRmRpYWdyYW0lM0UlM0MlMkZteGZpbGUlM0UlCf0tAAAgAElEQVR4XuydCZgc1XXvz63qWbSMQBubQGzqNg8BAYSM8RJw4CUE/Jk4CTgPgv05YPx4BNtgJKEF0WhFEohAwMTYTmIT8LN4jpMYHGwDxrFlLBaxE9wlQAgkQBvSzEiapavu+0713Jmamqru6p6u7lr+7c+fhp6qe8/9nVt37r/OPfcKwgcEQKCuBE4+f8UFQrOuI4s+RoLG17VwFAYCIAACINBcApK6SaPfSUu74+VHF/y0ucagdhAAAQEEIAAC9SNwyvnLvi8FXSSIJtSvVJQEAiAAAiAQNQKSqFNI+veXHl30hajZBntAIE0EIGbS5G20NVQCLGSEoIskhEyonFE4CIAACESFgCDqlBA0UXEH7EgpAYiZlDoeza4vAV5aRsL6ASIy9eWK0kAABEAg6gQ4QkNS+19YchZ1T8G+pBKAmEmqZ9GuhhI45YJlvyBJ5zW0UlQGAiAAAiAQDQKCHnvpp4v+ZzSMgRUgkC4CEDPp8jdaGxKBU85f1oVk/5DgolgQAAEQiDoBSd0vPbqoI+pmwj4QSCIBiJkkehVtajiBU/50mWx4pagQBEAABEAgMgRe+s9FmFNFxhswJE0E8OClydtoa2gEIGZCQ4uCQQAEQCAWBCBmYuEmGJlAAhAzCXQqmtR4AhAzjWeOGmsnkP/ahfS5Pzl1sID+okn/9NBTdM/9v6JrLj+bvnTxWdSS0T0reHPLTvrc//6W/TtVzoYXNtNVCx4Ydr27Ducvf/yzFyh/5yOD9/u1xKvc2luNO0EgXAIQM+HyRekg4EcAYgZ9AwTqQABipg4QUURDCLDI+OM/PJFW3PMoPfzEy8NEiRIZypCP/sExtPyGz9L6Z9+wxYfzo37Xva+XDp06YVh5Suh84ozjaeFt/0FPv7h58Fau/zPnnjwontQvPvNHJ9Ocq86jdY9stEUVPiAQNwIQM3HzGOxNCgGImaR4Eu1oKgGImabiR+VVEPjxP3zFvlpFV9StXt+XEzMcwbnkwtPphw8/R3/9uTPp5//12jDBw6LFS8yoMt96Z9ewaA7ETBVOxKWRJAAxE0m3wKgUEICYSYGT0cTwCUDMhM8YNdSHwH0rLqPTTzpqRGTEq/RyYobLmTppvC2KvIQQxEx9/IVS4kMAYiY+voKlySIAMZMsf6I1TSIAMdMk8Ki2JgIsPo6bPmXwXmcejLNAPzHDUZQF15w/GI3hKM1lf/ZReuDfnh5cIuYnZrDMrCaX4aYYEICYiYGTYGIiCUDMJNKtaFSjCUDMNJo46qsHASVWDplcOh4jaM6MW6h4LR3z2wDAudmAsw1YZlYPj6KMZhKAmGkmfdSdZgIQM2n2PtpeNwIQM3VDiYKaRICXjR171ORhCftekRm3AHKau+9A3+BGAG7Bo6I5H+zoHJGvw2VAzDTJ8ai2bgQgZuqGEgWBQFUEIGaqwoWLQcCbAMQMekYcCLiXhzlt9trlzEvMeC0p43LUls4PP/7y4LbL7g0A1DUbX3lnxFbOEDNx6EGwsRwBiBn0DxBoDgGImeZwR60JIwAxkzCHJrg5XhsA+O0w5iVmvCI4CpdzIwC/nBm+/8xTjxmxpA1iJsGdLiVN8xMzq+au6qDW4lXzli28PSUo0EwQaCgBiJmG4kZlSSUAMZNUzyazXV75LO58GW65W8yoyM4rv982IrLC1zuT+3mnM6+tmfk6Fj1HHTFx2I5qEDPJ7GtpapVbzLCIEWP6F5AU3yCS5twli8akiQfaCgKNIgAx0yjSqCfRBCBmEu1eNA4EQAAEKhJQYmbVqlUdYj+LGLpOEglBwiSy5s9dctOdFQvBBSAAAlUTgJipGhluAIGRBCBm0CtAAARAIN0ELjunZcJwEUOtA0R2zV2yaGgv9HRjQutBoO4EIGbqjhQFppEAxEwavY42gwAIgMAQgb+eTT8iEp8lki3gAgIgUB8CQtKTc5Yu+nS50iBm6sMapaScAMRMyjsAmg8CIJB6As7IDBHx/EpFZnbPXbJocuoBAQAI1EBg9eJlcu6SRWX1CsRMDWBxCwi4CUDMoE+AAAiAQLoJeOXMDIgak0jMn7tkIXJm0t1F0PoaCEDM1AANt4BALQQgZmqhhntAAARAIDkEfHczs+gbJAi7mSXH1WhJAwlAzDQQNqpKNwGImXT7H60HARAAgfLnzPRdNW/ZTThnBt0EBKokADFTJTBcDgK1EoCYqZUc7gMBEACBZBDwEzPJaB1aAQLNIQAx0xzuqDWFBCBmUuj0GDX5vhWX0ZmnHjNo8ZtbdtLn/ve3Bv9bHY751ju7PA/DbERT2YZDJnfQw0+83IjqUAcI1J0AxEzdkaJAECCIGXQCEGgQAYiZBoFGNVUTYCFz7FGTaeFt/0FPv7jZvt/93TWXn00XfPok6usr0sp7fzZ4XdWVjeIGtmnbB3sof+cjoygFt4JA8whAzDSPPWpOLgGImeT6Fi2LGAGImYg5BObYBJRI+b8/eZau/us/pHFjWomjMjt2d9PUSePpxf9+1xYPP/6Hr9g/n3D8YfTrZzbRPff/yr73SxefRX39Jn24dz/99Jev2GW6v/vIcYfSGaccTfv299oi6E/+8ERqyeh2PRz98SqH6/7cn5xql7fvQB/95y9foT/99EnU2qLTPz30FP3B/zjSjiT1F037v9/eupu+9qVP08SDxtLDj78MwYP+HUkCEDORdAuMijkBiJmYOxDmx4cAxEx8fJUmS1W0g8UBi5WfPvkq3fy1C21hwoLiiEMPpv947CVbKNz5T7+kM06ebgsJFiH/964rbGHDQmLOVefRukc20qdmzxjxHV/Pn6sWPDB4D4shVbcSSM5yOrt76P4fb7DvU/VwORyZYaHFUaJb7nyEZp9ytP3zz//rNfrMuSfbNmIZWpp6cLzaCjETL3/B2ngQgJiJh59gZQIIQMwkwIkJbAILhdffeN8WKGr52Pdu+yI99NON9NnzTrHFA39UlERFSjiS8+mP5ei7635riwcu540tO+nEGYcN+47FjhIhLJSW3/BZO+9Fff570/vU1poZcQ8LmwXXnG9HilT0RZXD97LIYnHEeTTzr/4T2vjKFjr9pOlNWwKXwK6BJoVAAGImBKgoMvUEIGZS3wUAoFEEIGYaRRr1VENAiZlPnHG8HdU4etokO9Lxqw0FuvDTJ9GPHn3BXhbGkRqOpvDHHVF55qW3B6M5KjLj/E6JEF6upqIsqixn5EXd88Jr7w6Kou27ugbLrhSZOedjOYiZapyPaxtOAGKm4chRYQoIQMykwMloYjQIQMxEww+wYjiB/NcutCMnvHSLc1A4P4XzUvjDuSj8UUu61OYAKs/mhdfesYUOf/bt76N1jzxn/8w5M87vnGJG5cdwzgx/fvyzF+y6ve5ROTEf7OyiZwY2JuClZH45M1dc8nGIGXTwSBOAmIm0e2BcTAlAzMTUcTA7fgQgZuLnszRYrLZc7t7XW/VWzCrKUiky44zCeDH1KqfSPWnwDdqYPAIQM8nzKVrUfAIQM833ASxICQGImZQ4OqbN5N3Kjps+ZdB6ladSTlQ4oyzuncmcu5VVQuJVTqV78HsQiCMBiJk4eg02R50AxEzUPQT7EkPglPOXdZGg8YlpEBoCAiAAAiAQnICk7pceXTS0+0XwO3ElCIBAGQIQM+geINAgAqdcsOwXJOm8BlWHakAABEAABKJEQNBjL/100f+MkkmwBQSSQABiJgleRBtiQeDk81dcQML6gSCaEAuDYSQIgAAIgEBdCEiiTpLa/3r50QU/rUuBKAQEQGCQAMQMOgMINJDAKecv+74QdJGEoGkgdVQFAiAAAs0jIIg6paR/f+nRRV9onhWoGQSSSwBiJrm+RcsiSoAFjRR0ESI0EXUQzAIBEACBOhHgiIyAkKkTTRQDAt4EIGbQM0CgCQR4yZnQrOvIoo9hU4AmOABVhkaAj4+Z0EbU0U40oX3g3zaiiWOJJBHt2U/U2UvU1UPU2TP0b9EKzSQUDAKNJyCpmzT6nbS0O7C0rPH4UWO6CEDMpMvfaC0IgAAIjJrAunXr9Lde+n2OiLKkaVmNKCtJ5ohElqScLAQVLCkNoYkCWcLQdVkwLfGXkqweIeUjRHpWCiunCS0rJd9HWSK5i6QokBCG0GSBLDL6Nc3oo75CPp+H1Bm111AACIAACCSTAMRMMv2KVoEACIDAqAnctmzZ0WafyJGUWU2jrLQoR4KFB2UFUcEiMgSJApFlCIsMU28p3HjLjVu8Kl6zePkyi6wD85bctNzr97ctXHa0bJFZyxQ50igrBuoSJGZYJA1y1CVJK0ihG351jbrhKAAEQAAEQCA2BCBmYuMqGAoCIAAC9SewJr/mELO/Nyd0ORBhEQOREluw7JSSDBLSEFIUTEGGsGThQMY0qo2WVBIzfi1bd/HF+paZp2XNopnjKJDgSI4QHMnJSaKJLKpY6JAgQ5NUKGpkZKi9cEP+hp31p4USQQAEQAAEokYAYiZqHoE9IAACIFBnAqtWreqwuntzGV7OJXQWAoPLwqQkKYQokLQMIbSCKU1D0/WC6G415tw2Z1+9TKlVzJSr/558fvw+asmRJbOCWIQNtYuILCI5EDkSRljtqhcflAMCIAACIFAbAYiZ2rjhLhAAARCIFIF8Pp8ZU9SzmiZyUnIuC2VpYKmWM4LBy8J42ZYmtYLQ+4wb8vmGRDDCEDPlHLAmnz/E7M/kdF1mpRT28jjO6xF2Xg/tIEEFjubYeTqaZcgiGXOXLioIIXifAnxAAARAAARiQgBiJiaOgpkgAAIgwATuyK88pr9YtPNKiLSBCAtlhaTj7Qm6JENoVLAsXnplGhlTL3xj+aJ3mk2v0WKmXHtX5vPHaFZLVpCVUwxLmxjQcfaSNSkNaW9eQIamy4LWL4woMGy2D1E/CIAACESRAMRMFL0Cm0AABFJN4O8WLD+0TzdzvOsXaZIjCZx0X8plUVEFO5eFJ9yWoWf0wvU3zzeiHFWIkpjx61zu6JYUlNM4osN5OlIcpHZp44iO0LSCME1jzJhM4W8XLNiV6g6LxoMACIBAEwlAzDQRPqoGARBIL4G78vkJfWZb1hJmzhYrmr2tsdqm2CwtByNDyNJyKF1mCl16Dyfe748jtTiImXJc8/n8hPGmlpW6npWmzA33FxUl+4ojY3ZEjP1lFbp0K7b+imMfg80gAALpJAAxk06/o9UgAAINIPCtq77V0nXErsGduDhBXdjnsXCEZehNvzqPRZBp9I/JFBYk8E1/3MVMue7ijKR5nJ/zwfDzc4TRr/UbC/J53oUNHxAAARAAgVESgJgZJUDcDgIgAAJrFi091tS0nBg4j4UjLbK0LOxYjrBIkgZvcayJ0mGQVqa1MC8/7900kUuymCnnR+4bUuczekTOeVaPIHEM9ws7R4dEQZBlWKQVSMsYaesbaXoO0FYQAIH6E4CYqT9TlAgCIJBAAqvzqw8js5iVwsxxsjgNP71+4O27Oo9FGBbevg/rBWkVM36Pgjtqp87P4R3XiKjDeX6O5KWGGhktNLZwff763Ql8vNAkEAABEKiZAMRMzehwIwiAQNII3Drv1oPEmGJWM60cn8ciSdqHMxKJrJTUXzqPRXKifcGyLEMKabROHF+4/vrrDySNRb3bAzETnKjqh8Lig0tFrtQPRU7Y5+hQnzo/xyJh8AYQWkYr6HvHGNffgX4YnDKuBAEQSAoBiJmkeBLtAAEQCEQgn8+3tptaVhfa0HksUqgtjjvsJG57pzDipWF4Ix6IauWLIGYqMwpyBUcIrf4+1/k5lBOlne7e4+25hd13S+fnEJ+fs+wmXs6GDwiAAAgkkgDETCLdikaBAAiszK88rsWystbAWSIk+PBEe9J3tJ2nMHDqPWlaQfIOVFq/MS+fT1UeSyN7CcRM+LTdfX7wkFCS09X5OWSfnyMMoVsFvV8Y1y9ftDV8y1ADCIAACIRHAGImPLYoGQRAIGQCdyxYfrjZKrOWKXIjzmMheo8Ib6lDdkHg4iFmAqOq+4UcjRzX15KjjC3os67zc8bZyyalZUcjNU0rkGkaxbEthfnz539Yd2NQIAiAAAjUmQDETJ2BojgQAIH6Ergjnz/YNNuyJp/HwlsbC41Puy8dZCiot3SQ4cAZH/yvJgx9707j+jvuQB5LfV0xqtIgZkaFL7SbS89X6fwcy7Jy2rCNLajHfX6OJa1CT+n8nJ7QjELBIAACIFAFAYiZKmDhUhAAgXAI3HXtXW29k7uyZMmsNZBwP3jyOolxvERG8rbGvL3xwJtjXbeM6/N57OwUjkvqXirETN2Rhl7gHcuXH97fwy8R9Cyfn8MvEtTBrpJoa+lAV8kvEArCKp2fMz+f3xS6YagABEAABBwEIGbQHUAABBpGYPWipcdThrLSLJ254TiP5ajSeRul0+6FlHbiPdb0N8w1oVcEMRM64oZW4H6WpUUcKeVd147k3db4eeYzlogsQ0q9IPU+zknb1lAjURkIgEAqCEDMpMLNaCQINI7Aqnz+CGG2Zi1h5nTi9fkDb3Mln81CW3m3JZLCEJoceJurGfPz8/E2t3EuakpNEDNNwd7wSu+66662/Tv25IQuspqlDSwHLe0WKIjG2C8s7A04hnYLbKVxhevy1+1puLGoEARAIBEEIGYS4UY0AgQaS2DljSsnau1mVrcoawnKcYTFfisrKSeJDthvZIU0+LR7TdcKRbNoYJ19Y30UtdpWL17+GAnZO/eWRRdGzTbY0xgCatzgc5ykxi857EiOOj9nv9f5OW27Ooyv/v1XextjIWoBARCIIwGImTh6DTaDQAMI5PP5dj6PJaNnshYfImkvC+PDI2VOCH7DyktJWLQMvWG1enRj/q3YAakB7oldFWsWL73VIu3DeUsWroqd8TA4dAKr8quO0KgvRxZv7iEGlqxRjkrn57xjb/RhL0Eddn7OG6EbhgpAAAQiTwBiJvIugoEgEC6BlfmVM3zOY5mmzmMR0jJMIkPD2vdwnZHg0lffvGwNWfTB3KWLbktwM9G0EAisza+cYTnPjBrYJIRIHmFvDiJLmxDw+TmaLgt6nzCuW7GQt2bHBwRAIAUEIGZS4GQ0EQTWLlw2zWyRWbJKuxLx8g4+MVyS/ebzHTGQeG9ZZAhdFgZODcdbT3SduhFYvXjpWiLtnblLFt5Rt0JRUKoJ/FM+376TWnLSkvYyVz4Ut7QLor30tc0+GFfwBgSlXRAFn58zrrVw44037k01ODQeBBJGAGImYQ5Fc9JLYG0+P8k0W7OkS/u8CP7jrkmVgEv7eFmYtHcXGshlyWgFrEdPb39pdMtX3bzsTs2iN+YsXXRXo+tGfekjoMZDi1/ekMwOnZ8jsoJkt8W7rfFmJPwCR5BhDp2f05c+WmgxCMSbAMRMvP0H61NGYO11140xD5qS5TeRvDsQkchqdoTF/rlNLQuTfJo3CUOXekHXe43r8nnsFJSyvhK15q5evPxuIuu/5y656Z6o2QZ70kVgVT5/JFktWcF5gPaLn2Hn57ztPD/HtIQhS+fnvJkuSmgtCMSHAMRMfHwFS1NEYPWipVnneSyytOtPVq0RV+exYI14ijpFzJu6+uZl90qLXpy3dNE/xLwpMD/BBHjstTQtp0uZ5U1PHOfnHDZ0fo4s2JEdqRdI7zPm5vPvJxgJmgYCkScAMRN5F8HApBIY8XZQalkS0t7emAS9rc5j4UPneClESyZTuC4/H28Hk9ohEt6uVYuX3aeRfGbOkpu+nfCmonkJJLB27doxfR9259y7O5IdFacWdX6OJGEIkgWpkTGGioWv5vOdCcSBJoFApAhAzETKHTAmaQRWzF8xWW+zsrzdqFDnsQwk3mtEXe5127q0Cp26ZeTzeazbTlpnSHl7Vt+87LtkifVzly78x5SjQPMTRoDH+bY2K2uaIkea5GXAORJqCbDo5BdSfPaWRcIgyzL0jF7o2DbZ+Mp9X+lPGAo0BwSaQgBipinYUWmSCOTz+bHDzmPhcxGEvSQsS0St6jwWwX/IJBlFMg2BHXWS1AXQlgAE1ixe9s8W0S/nLVn0vQCX4xIQSASB2xcuO8oa2EmSNyMQ9pLhwfNz3hp2fo6wDGGSMWfZTW8lovFoBAg0iADETINAo5r4E1iRX5Fr4S1A7TdvIke8pppPvSfhWEvNu4VZhmkKQ2tpLczNz8Va6vi7Hi2oA4HVNy29X+jaz+bkF/5LHYpDESAQawKSSKzNr8iaRTNHmsYCh1+A2Zu6EMlDRpyfo8lCb78wFq5Y+EGsGw7jQSAEAhAzIUBFkfElMPQWjbKWFHzS/cAfGftN2lscWREaFfg8Fimwy018PQ3LG0FgTX75KXPyC1/iulYvXvYgEf1k7pJFP2hE3agDBOJKYM0NN4yT4ydmaWDXSo0FztD5Obo6P4c3gBG6KBRN09DGtxXmzZvXFdc2w24QGA0BiJnR0MO9sSRwWz4/RZpaVup6VvK2nJKymr0sjLc4Hrm+WQppTDzy0MJXvoL1zbF0OIxuCoE1+eUnSUuul1L+nCQtIkFLNE37kSXpQyJ5GxFtnbdk0QVNMQ6VgkBMCfDfr2J/Jid0ymrEW/QLftGmdrvcwy/cSBs6P8eyZOFAxuQ8zGJMmwyzQaAiAYiZiohwQRwJfCufH7uXWnL8ZovFihSUK53HYkdaMvZ5LPx/SQUSZGDnmTh6GTZHncDqm5f/O0l5AUmSJGg3kewlKSaSIF1o4itYchZ1D8K+OBG49eabpwvZmhXEB4WWcjel5L99ImuRfEOdn6NposDn51il83M2x6mNsBUEvAhAzKBfxJaAlFKsvmlZTmT44LOBQ89KOSwj1xwLYZApDJwJEFt3w/AYErjtpiWzLNJ+TYLGuMz//dwli06IYZNgMgjEjkA+n9fGFPWspnGuJ2WlfViofQQAv9ybol7u8ZbS6vwcofcZc/L57bFrLAxOJQGImVS6PbxGr1m89Aqpac/PzS/cWK9a1NsmTrgXQuZIltYP85snQfQmD752LosQBcJuMPXCjnJAoC4EVi1e9rAgutBRWKcQ4stzblm4ri4VoBAQAIGaCeTz+fEdppa1NF52Le2tpYnPPCudnyP4/BxBZFgk7e2lSRPGOOovXJPPd9dcqevGNTcvv3/OLQsvr1d5KCd9BCBm0ufz0Fq8evHS24jEV4nov+YuWXReNRXdlr9tSpF6csLidcAiK0sDqRIse6TkhPtS8r2w+F/NKGzbbNx3333Yp78a0LgWBBpMYM1NK84iYf1cEo0fqPqVuUsWndxgM1AdCIBAlQTuWrFias+BYo5IH9jFU+Y4v7S0XFvwslGDSBQkv1AcOD9n+qvPG5c89JAZtKrVS1dnqa/vv0nQmyS1L81dumB90HtxHQgoAhAz6AujJrA6v/xUacl/FEQ5QdRPmrx4Tv6mx9wFr1mzZlyx80AuwwOjvTOLNRDmtpPvNbLPYbF4cLR3aBGmaXSXDpDECcqj9hIKAIHmEVh987KfkaQ/JqIuackvzFt20781zxrUDAIgMFoCty1cdrRskVnLPiiUshovXxs4P6d0QKg0OB9VSK0gNNMQ/cK4Yfmit73qXb146dNEYjYR9QkS+TlLFq4crX24P10EIGbS5e+6t3b14qVfE6TdIUlyOJo/e4ta8WMtVmYg8d65LExMKb3JoYFBThaEEAb2zq+7W1AgCESKwJr80nOkpf1UkHx9zpJFp0fKOBgDAiBQNwLr1q3Tt7y2afD8HCGl46WlnGQvWZPSEJoo8NbSui4LpknnCEHzB6K3nULQK/2i+KUF+XyhboahoEQTgJhJtHvDa9zahcumFTP0HSnpk2Jo+QhXaBHRG/b2kEIaQ7umaMb8/HzsmhKeS1AyCESawJqbl/+QyPzRnFsWI1cm0p6CcSAQDoFVc1d1iLFF+/wcYW8pzVtL85Jy7USSfIwodThq7idNWzI3v2BZONag1CQRqF3M/O2/nUxtmW+QJS8g0+Q37urNfJL4oC0+BP5HSxd9cdwWKkqNWgXrl9KnV2r0owNH0At9B4Fdkghooo/0zGvU338v3fW5+2puGsaNmtHhRhCIHQGMG7FzWTMM/lrHmzRNPzCsap5L7Jc6rezk9Fl8UkWghnGjNjFz7Y+/TppYQ8cel6FDDiEaOzZVnNHYEoGO/n00c++bdPru12n6vvepKHRqtYrUp7XQ6plfoM6WcUCVFALFItGePUTvbNlD3d2bqde8kO6+aFtVzcO4URUuXAwCsSeAcSP2Lgy7ARduXU8f3/EitVhF6tFbyRQ6vTxxBr14cJbe6Dgy7OpRfhQJ1DBuVC9mvvrjq6itfQ2ddtoEiJgo9oLm2DSueKAkbHa9Tsd1b6XN4w+nb+Yubo4xqDVcAps3s6gp0O2f+UjgijBuBEaFC0EgkQQwbiTSraNp1Iyud+jKTf9G+zPt9OLBOXpxYpY2jz9iNEXi3qQRCDhuVCdmrn54IrUV36dZZ7RSx4SkIUN76kSg3ewlTUp7gMInoQRefXUfbX/vTrrzzxdWbCHGjYqIcAEIpIIAxo1UuLmaRmY7t5AxYXo1t+DatBEIMG5UJ2Z4mchhhy2jmTOxfihtnQntBQEngf37iZ7Z0EV3XFT5rQbGDfQdEAABJoBxA/0ABECgWgIBxo3qxMwNj/yMsrk/pkMPrdYUXA8CIJA0AuvXd9KB3k/S3X/2ctmmYdxImufRHhConQDGjdrZ4U4QSCuBCuNGdWLmGw8/TyedfCpNnJhWnGg3CICAIvDs03toT9fn6O7PPVkWCsYN9BkQAAGMG+gDIAACtRKoMN+AmKkVLO4DgbQTgJhJew9A+0GgegIYN6pnhjtAIO0Emi1mvnNqO11xdMugG7qKkq55qZfuf6d/mGsuP6qF7jmljToyQ8fVPL7DpPN+u3/wusc+PpY+OlHzvJ8vcv+e//vcqfqILrC1R9IXnjtAT+w0ie07/9DM4H+ri/9oik7fnzWGXu+yaK7VeK0AACAASURBVEWh1/55Wrv3UTqqTV88qsWzPi7ztS6LZj6xr2J3ZJsPbxcVr3W3zY/rq380jk7s0AbrdbZdfbnkhDaal22l1qHL6Ltv99OVL/RUtNd5gWI2IUMjfKT8+84BOaJt3JYTOjTbB/zxY12LTapfcD8od7+bpxcnZ1mq3e7rgvQn7tPu58LJ0c9OZaPf7xV/Zz9Vz1A536jfsQ3quQjk+BAnJRg3Sh7AuFH6u4Bxw7L/FmLcKD8yYdzAuMEEMN8Ymr+mZdwINTLDE+mjxohhE1uekH1qsk6rjD5a/Hqv/eSpAcgpXrwmWEqs7OiV9OUXemwxoj6qDOek3k8YsF0HtQh74nbpkS0VxYxTUKkJrZfgCCpEyg3Hlcrwm5Sq9jsnutxO/jhFlNsnLGS+fnwL/d0b/YP+UOLm17uGi8lKE1y24aLDM/ZlL+4t/fFVH6dYdU/GvcTMox8Uh4kpZROL4GpEFte79qQ26uyX1KaXfO7sN34TeTenoNdVI2a8RDTfzzY7nw81OJdrh5fP3AO63wDPdV4yLeP7ksDX7yGJGYwblZ60kb/HuFF6CYJxY+jvKsaN0gtOzDf8xxOMGxg3kjLfCE3MlJsgOSfZaoK1buvwySs/fs7oCE+M1aR3b7+kH20rDk6++VpVplM8+T2oPPG7+tgWuv6VXjp7sh4rMeOc+Dsn5UoUqkkp/ze/0XRzdfP2K69cPX5DI/vgvR5p/1pFWpSNTlExtW2kwHVHZtyTEnf73JE9P5uUuPh/2/rpb6YPF21KmLptLdf3/MQQR/DUG5BKkb5qrnOKdS7Xqx3u58TJwi1W3aK33PNXcUodgpjBuFGRuucFlSYlGDeqE+sYN4a/5MK4gfkG5huVx2aMG80bN0ITM15RAa+uUOmtsPNN94Jcmz1J5ojB1FYx+OafJ2QrT2yjjXtMOmeKPviGOWliptyk1TkBZyHw4Lv99ptK/pRbOuQV0an8yI68QomVe98qLR90R3vU7x94t5/+8ogWYkGqIkZBIjPOt4xch4rqVbJVCSy1VFCJDi/B4ldWJe7O+0YbmfGrq1w7vKJr5bg4Ix83ZlvtS4MsgRxRZghiBuNGpR7t/ftyYqZS/1W/x7gxxBbjxsh+hnGjtNIA841S38C44f2M8AtdzDeGj6UqyBDmfCMUMVPpj6ezC1SKADijKJyTwmLm9k19dOXRLXTtS6WlZjyBPGuSTk/tNoctl/H6A69sU5PpoJNPt81+y8y8cnT43qD5HuUmJUHeoqs/wvz23ysPycuOoPki5aZZbj+6J6VuscM5OmrJWFAx4xw8gyw184pKOJdTBeGp/nh5Rbm8eATtT37XeS0zq9SOSi8E3HYqju0a2XlSXjlsgabUdRYzGDdGUse4UVomql6SYNzwzvHEuFHKb3UvCXc/UZhvDCeC+Yb3MjO3WMN8YyiyHdX5RizFDEca1pzURo+8b9pv6HnizEKGP87JqnuSrh7jajcAiErOTJDJt1PMOIctZ5K/32YBfL1zwwD3Bgx+E1wvkeHu8E4xwz5T+U88kVYi1bkBgNcys2rFjHugdvMLwjNMMePcGEOx7bNoRL5MpXZUO7hwXbXmIA3rAzETMxg3Ri7lVf7EuDHUsys9bxg3hl5EBXrp4b4I40bZDYcw3yjfqzDfGOJTzcvTpM83QhEzalLM/1ZavlJpIua1zEwl7nM05lajz15iNv+1Uv6LW8xU2hksaGcIGpmpVF+lwb9ey0X83iQEfftdyS/Odqilal5tU4LILWacETIOywbJmXGXUY6lV1TKLWb5v9WOdeXe6AVlxuUF7U/u6/yS84O045wpmRHL+ir1s2pY+pZV50kJxo3Kux36+QLjhv8b1mr6epDnDeNGKVIWdKnviD6LcSMWYqbS370gLxcrlaH6BuYbQ09JUGaYbwwfWUITM+U6pzPUO22M5pmozmb6bQDAYobvU3kyx4/TbNHkrrNSUqx6Q602A3AmlZd7++ZXbpD6Kk0yK5URNJHXLeyc9aolYLxMz28yX00ehl+eg3PXOPaXWi6i/giq6ABHinqsUm6PmiiMdgMAv/7nXo5RjqezXUGvcy6LLNefvESP145kQdrx5M6irx/9+nE1E7xGihmMG5VGCO/fY9zwFzPVTpS8dvbDuFHqdxg3TMJ8o9QXnM8V5hveO4Ji3GjcuBGamFFvWb22Znaf+aHe7AfZmtm585RaEqXWldciZtw5NGqq4FwG5d45q5liJujWzH5bCVfa3Uq1P2gidjnR56zL2GeNEDNcl1oKqJb++YmZapdF+dnvFsjltlyeMU4bXPJV7XXODQ5UO51nJPlFcBQP1aeDtqPc1sydxZGbQER1UoJxIxwxg3Ej2JbuQZ+3ascD9qpzIxaux2t8wbjh6P/fePh5OunkU2nixEAPhd+W7phv+OOr9BIE4wbGDa/AQhTnG6GKGedkVT1Ooz000ylm3OKlFjHjnLxXOlzSKXSq3QCgXJ6Kc6jxy/NxH57nvi7ooZle13kdmhk0X6ac6HMKRV4O6I7MOB8S9cdeiRn3AaVeuSR+Q3Ql4eNlszNXiMv1OzSzXtf5iRk1meeJzoYPTTpzoj64UYK7ve52eC2R8fNjlMUMxo0hT2PcwLjhFEEYN8rrmqB/F4OMle5oPOYbQweQY74x1A8x3/A/6F5RasR8I3QxE+iVCi4CARCIH4EQ1r7HDwIsBgEQqIoAxo2qcOFiEAABIqowbkDMoJeAAAjURgCTktq44S4QSDMBjBtp9j7aDgK1EYCYqY1bGHe5lyi56wi6tCsM2/zKLLdTGd/jtxwrTBu9lsU56wu6NCdMG1NRNiYlDXEzxo36YMa4UR+Ooy4F48aoEQYpAONGEEqVr8G4UZlRQ66AmGkIZlQCAukjgElJ+nyOFoPAaAlg3BgtQdwPAukjADGTPp+jxSDQEAKYlDQEMyoBgUQRwLiRKHeiMSDQEAIQMw3BjEpAIH0EMClJn8/RYhAYLQGMG6MliPtBIH0EIGbS53O0GAQaQgCTkoZgRiUgkCgCGDcS5U40BgQaQgBipiGYUQkIpI8AJiXp8zlaDAKjJYBxY7QEcT8IpI8AxEw6fO7edcy9y5jz9+V2++KDLv/+lHb6j/eL9NnDMsSHXd7/Tv8wiOoar99VS5sPU9p6wKIndprV3orrm00Ak5Jme2DU9WPcGDVCFFAtAYwb1RKL3PUYNyLnkuQbBDGTfB+7TybmFju/y47T6OpjW+j6V3ptYcK/O2uSTjOf2DcCTiPFTD1FUfK9HMEWYlISQacENwnjRnBWuLKOBDBu1BFm44vCuNF45qgRh2Ymvg9wZGPliW10+6Y++saMVprWLujNfRbtKUr6sG+o+Zv3W3TlCz2DX6iIyKVHttAVR7fY33/37X568N3+EZGZG7OtdGKHZl/zWpdF177UQ98+tZ2mtgnqyAhS5+M439Y4y5oxTqNf7zLp8HYxrJyndpt23RxFumNTH103YL+KKrFtf3Z4hlo1omteKgkxfCJEAJOSCDmjOlMwblTHC1fXkQDGjTrCbGxRGDcayxu1OQggMpPs7sAC4pixJaHBn/N+u5/4sCwWCtt6JF14mG6LGreY4WvVwDT/tV77XhZFLBh4eZlaZsb/soDh7/l6Fjb/+HY//Z9jW+iBd4v05M6iLX5YrHzmsAy5y7rkiMzgddPGaCPK+ZujW+ylbF88qiSo2P7HPj7W/plt9osgJdurMWkdJiUxcdRIMzFuxNZ18Tcc40ZsfYhxI7aui7/hEDPx92G5FvDEf0efpFMmaIP5Lf9x5hh6Ya9FR7SLQaHjFjM8KCmxM+vJ/XYVz50zll7tlDTrYG1YzszZk/XB6I2KoigRwiKH79tXJBqXIfIqS+XWOCM37nK+fnwLPfK+SYtf7yU+cZdF2PN7LNt+Fjj4RJAAJiURdEowkzBuBOOEq0IggHEjBKiNKRLjRmM4oxYPAhAzye4WanD56MGaHQEx9ll2hIWXeXH05N63Skuz3Dkzl0zL0IPvFj2jKc7IjNFt0VFjhS1SWGRcdmSGvvlW8MiM2kSA83ZYoLjLqRSZgZiJcP/FpCTCzilvGsaN2Lou/oZj3IitDzFuxNZ18TccYib+PizXAiUwnt5j0V9Ny1CfRdRrSTuXhQWNypPx283ML8/FucyMl5dxLs7OPkmd/ZLWbSsSLx8LmjPDkRnesez7s8YMKyf/+z572dpBLcI3ZwZiJsL9F5OSCDunvGkYN2LruvgbjnEjtj7EuBFb18XfcIiZ+PuwUgs4R4YFwReeOzBsi2P+nj9eu5ZVKhO/B4GKBDApqYgoyhdg3IiydxJsG8aNWDsX40as3Rdf4yFm4uu7aizn8O+5U/VhtzgjM9WUhWtBIBABTEoCYYryRRg3ouydhNqGcSP2jsW4EXsXxq8BdRUzNzzyM8rm/pgOPTR+IGAxCIBAfQmsX99JB3o/SXf/2ctlC8a4UV/uKA0E4kwA40acvQfbQaA5BCqMG6sXL5NzlywS5Ywb+uW1P/46HXbYMpo5s7R+CR8QAIF0Eti/n+iZDV10x0UTKgLAuFERES4AgVQQwLiRCjejkSBQVwIBxo3qxMzVD0+ktuL7NOuMVuqoPIepa2NQGAiAQHQIvPrqPtr+3p10558vrGgUxo2KiHABCKSCAMaNVLgZjQSBuhIIMG5UJ2bYuq/++Cpqa19Dp502gcaWDjfEBwRAIEUENm8memdLgW7/zEcCtxrjRmBUuBAEEkkA40Yi3YpGgUCoBAKOG9WLGbaal41oYg0de1yGDjmEIGpCdSUKB4HmEygWifbsYRGzh7q7N1OveSHdfdG2qgzDuFEVLlwMArEngHEj9i5EA0Cg4QRqGDdqEzPcsr/9t5OpLfMNsuQFZJpTiGTZxJuGw0CFoRE4PrOP/mrsVtpptdIOq5V2mm32z/Z/m20kQ6sZBTeNgCb6SM+8Rv3999Jdn7uvZjswbtSMLu43/mHbLtrQN5F6pRb3psD+oAQwbgQllfrrxosiTdH7aIrG/++1f56q9dF+qdO3uo9JPZ9UAahh3KhdzKSKLBrrJnDrzbdO10V/Tlp6Vgorp0nKSqIcEWUFiYJF0iBBhpBaQWimIfqFccPyRW+DJAiAQDoJrF68fKvU+mfPy+eri+ilExdaDQKJI3BPPj++x2zLmsLMkaQsaSJLUtrzhoHGGpJ43kAFnj/o0ip06ZaRz+e7EwcDDaorAYiZuuJEYfl8XhtPrTlpWVmLZE6QliXiwUrwv5MEkWFJaQhNFMgSBpFptI/JFL66YMEO0AMBEEguAYiZ5PoWLQMBReDiiy/Wz5x5WtY5BxBCZiWLFikmC0EF5xxA12Wht1czFqzEHAC9qHYCEDO1s8OdVRLI5/PjO0wtawrNfisjBeVE6Y3M4FsZEixySlEdXeqFdr3XuAZvZaokjctBIHoEIGai5xNYBAK1Erht4bKjZYvMqtUZ/DddI5GV9stLMgRRwXKszjBlS+HGW27cUmt9uA8EyhGAmEH/iASBFfNXTM1kijkiPUsaD5AypwmRlZJyJOQuIURBSmFIsgyNREFomrHh1eeNhx56yIxEA2AECIBAWQIQM+ggIBAvAnetWDG154DH32X7BaTYTbycnETB+Xe5m/oK+XzeildLYW3cCUDMxN2DKbAf+TkpcDKamHgCEDOJdzEaGEMC5fNYeGMnFivIY4mha1NlMsRMqtydrMYiPydZ/kRrkk0AYibZ/kXrokvAK4+FhCwl3w/ksZCUhnTkshaLmQLyWKLrU1g2nADEDHpEIgn45+fYmxHw7tGGOz+nS+/FrimJ7A1oVBQIQMxEwQuwIckEkMeSZO+ibeUIQMygf6SOgDs/R1gyR5yfY28tLXfZIgf5OanrF2hwuAQgZsLli9LTQaCWPBbkl6ajb6S5lRAzafY+2j6CgPvNFs7PQScBgfoQgJipD0eUknwCpZUFfuexII8l+T0ALayWAMRMtcRwfSoJeK45xvk5qewLaHRtBCBmauOGu5JJoGweC4nJvLUxrxCwhGUfPo0z2ZLZD9Cq+hCAmKkPR5SSYgLut2hD5+cgPyfF3QJNdxGAmEGXSCOBYdF+aeVIlD+PRfQL44bli95OIyu0GQRqJQAxUys53AcCAQg41zdLYeU0qWV5Fxnk5wSAh0sSRQBiJlHuRGMcBJDHgu4AAs0lADHTXP6oPcUEyufnUMHiHdckGUJoBaGZBt7YpbizJKDpEDMJcGKKm1B9HoteaNd7jWvy+e4UY0PTQaAhBCBmGoIZlYBAcALIzwnOClfGhwDETHx8lVZLA+WxCGFYEnksae0jaHc0CUDMRNMvsAoEPAm4T2t25+fYJzULWSBLGCTI0KVV6NItnJ+D/tR0AhAzTXcBDBggMBQVp6yUYiCPhQaW/5LByfeW4FPvERVHpwGBOBCAmImDl2AjCAQgMCI/R2hZySc8E2Vxfk4AgLgkVAIQM6HiReEuAu7zxKQlc1rpPLEskdhNJA0iUZBkGRqJgtA0Y/qrzxuXPPSQCZggAALxIgAxEy9/wVoQqImAehNpmSJHGmWHn5/jzM+RBaER8nNqooybyhGAmEH/qDeBUh6LljWFliNJWdKkHWkRpRc4gsWKIDKkpAJHqqkojHGt/QXksdTbEygPBJpLAGKmufxROwg0lcC6iy/Wt8w8LWsWTVvkCNJ4EpAjEvzvJJ4IWFIaQhOlpWtkGsViprBg5YIdTTUclceOAMRM7FwWCYMr5bGQvbRWFOw8Fk0rkGkaQreMOfn89kg0AEaAAAiETgBiJnTEqAAE4kmA83P2UUuOLJnlt55CUK60RIN46ZpEfk48/dosqyFmmkU+HvUGzWMhiwxNlwXs7hgPv8JKEGgEAYiZRlBGHSCQMAJr8vlDpNmaJV1mpWXlNOTnJMzD9W8OxEz9mcatxGryWFi06Bm9MP3VGcYlD12CPJa4ORv2gkADCUDMNBA2qgKBNBBAfk4avFx9GyFmqmcWxzvK57GQIKLheSyaMMYR8lji6GvYDAJRIQAxExVPwA4QSDiBdevW6Vte2zSQn6NlB5J0kZ+TcL+r5kHMJMfR+XxeG0+tOWlZWYtkzs61kzJLws63mzwyj0UYQu9DHktyugBaAgKRIgAxEyl3wBgQSCcB5Ock3+8QM/Hz8d/dfOv0ftGfk1b581iQxxI/38JiEEgSAYiZJHkTbQGBBBKomJ8jRYGEMJznRXRTXyGfz1sJxBHbJkHMRNN1nMfS1mZlTXvbds6BkzkSYiBy6jyPhQyyrFIey4kzjEsuQR5LND0Kq0AgfQQgZtLnc7QYBBJDwJ2fIwYmYnxQqH2KN5FBkrduLZ2fY8qWwo233LglMQBi1BCImeY5S+WxWJpeEivDzmNBHkvzPIOaQQAE6kEAYqYeFFEGCIBApAhwfs5bL/2et5DOkqZlNaKsdJyfw9tKk+P8HF2Xhd5ezcD5OeG5EWImPLZcsjOPhc+MkpbI+eWx2DktUi8gjyVcn6B0EACBxhCAmGkMZ9QCAiAQEQKr5q7qEGOLWT4/hwWORiLrPD+HTw0nYRl8SCgfFqpZptGlW0Y+n++OSBNiaQbETH3cduvNt07Xzf6c1PjMJ22gD7NYt89/skW6tA+5JYOEaeimXrhh+aK361M7SgEBEACB6BGAmImeT2ARCIBAkwio/BwpTDuq4zw/RxDtlJIniCxypD1ZFJpmID8nmLMgZoJx4qvceSx8aC2LldIOgAN5LJIMKVi0WAaLmGNP+UgBeSzBGeNKEACB5BCAmEmOL9ESEACBEAlwfo6pl5buaPYyHpUoLWYQSYPzcwRHdcgyhEWGqSM/x+kOiJnhndMrj4WkliV7OeRQHotFwhAkC6QJQ+7PGPNWz+sKsZujaBAAARCIHQGImdi5DAaDAAhEicBgfk6GsmRpWSFZ5Nhv0HliOonzc3gzAhK8G5Qw0pqfk0Yx45fHIoQdaZmizmMhaZXEMPJYovRowxYQAIGYEICYiYmjYCYIgED8CAzPzxE5jThPh4WOnd8gk56fs/a6tWOuv+P6A+y5JIsZvzwWFrSS5CZ7Rz2NCpadxyIMvVUWbliEPJb4PdGwGARAIIoEIGai6BXYBAIgkHgCzvwc3m2NhJaVHNUpbSs9Ij+nX9OMvhidn3PrTUsu1Uj7Z9Ion+ncdUexY8omqfXPFmbLBULI24nkt+YsuWluXBxdVR6LZhlURB5LXHwLO0EABOJNAGIm3v6D9SAAAgkkcNuyZUebfc78HLKXrgkSMyySdsJ3HPJzVi9etpOIxgoik0jokmSnlPIgQaLF1PUT5ufnb4qS+3zzWEpLB/l/BcGbQBAf0oo8lij5DraAAAiklwDETHp9j5aDAAjEjEDc8nNWL156NUlxOwka40DdK4j+Zc6SRVc2Az/nsbRSa67FsrLO81iQx9IMb6BOEAABEBg9AYiZ0TNECSAAAiDQdAIqP8cyzZwudN58ICt5IwJJOZIkibfxDfn8nDU3L79/zi0LL3fCWL142ftEdKj6TpDoL2raic6ozJrFS6+Qmvb83PzCjfUC6ZXHYm+37ZPHYmn9xvx8fnO96kc5IAACIAACjSEAMdMYzqgFBEAABJpGYE1+zSFmf29O6DLL+TmShJ2bU8/8nNVLV2epr++/SdCbJLUvzV26YD032BWdGRGVWb146W1E4qtE9F9zlyw6rxpIZfNYJH3Iy8JKu8iRIXVZ4DyWAxmTD0AtVlMPrgUBEAABEIguAYiZ6PoGloEACIBA6ARW5vPHaFZLls/PEULmSIpsrfk5qxcvfZpIzCaiPkEiP2fJwpUlQaOiM6LfHIjKrM4vP1Va8h9F6TDIftLkxXPyNz3mbnDFPBaSA/lDwjClaWi6XsB5LKF3G1QAAiAAApEhADETGVfAEBAAARCIDoF8Pp8ZU9SzlNGzwrRynF/CJ9Hby9YETfQ6P8c06RwhaL4kGk9EnULQK/2i+KWMpZ9LpN1BUj4wd+miK1YvXvo1QdodkqQYaPHeoqZ9zM5j4dPuRWnzA2cei9r0wCLLkKYw9Ja2wpz8nO3RIQZLQAAEQAAEmkEAYqYZ1FEnCIAACMSYgGd+jn1yvXYiST4+hzoczesnTVsipTlTWPJu0rRFUtInRUnwqI9FRG/weSwkpKFpomBawkAeS4w7CUwHARAAgQYRgJhpEGhUAwJOAmfS8xf0txev04riY3px2KQOoEAgtgTO/fJv6eDDO4fZX+zTqe9AK/3nnWfTYdkd9PG/2khmUaNMizl4XbEvQxt/MpPeefXw2LYdhoPAIAFBZrGN3tF75PcEmWs30MeGPxRABQIgUFcCEDN1xYnCQKAygTNan/m+2U4XvXumnLDnWEn94yrfgytAIOoELmwz6OMtW6hFWNQjM2SSoJeLh9KL/YfRG+bEQfM7RC/NzOyg01veo+n6XipKjVqFSX1Sp9X7PkGdsi3qTYV9IFCWgDCJxm0XNPUVOjD591qv3k+f30Bn/BzYQAAEwiEAMRMOV5QKAp4EWMh0HkUXFT5rTQAiEEgKgRn6brpy7EbaLzP0YvFwerH/UNpsHlyxeeNEP83MbLeFzXH6h/Y939zP+wfgAwLJIDBxk6Dcw5pJUpy5gWY9l4xWoRUgEC0CEDPR8gesSTABXlrWN6H4g+evgJBJsJtT27RsZhcZxck1t79dFEkjSftlS81l4EYQiCKBQ14SdORvxbMbD8yGUo+ig2BT7AlAzMTehWhAXAic3v7ML975lDxvx0kyLibDThAAARAAgToQOP3bWldLt/ZpRGfqABNFgICLAMQMugQINIjAGZlnu178G3M8cmQaBBzVgAAIgEBECBz7C3Fg6iti7tM0++6ImAQzQCAxBCBmEuNKNCTqBM6kZ+WG64Z2cIq6vbAPBEAABECgPgSm/U6jaU9p+adp1i31KRGlgAAIKAIQM+gLINAgAhAzDQKNakAABEAgYgRKYoZueZpm5yNmGswBgdgTgJiJvQvRgLgQqFbMvDrvZDrx0DGDzeszJa16/D1a/Oi7I5r8nc8fS1ecOZUeNzrpvHtft3+vvvPjw9d+75mdtPai6XTv+u0jyn3s6hPohEPa6QsPvknTDmqle/7iaOpo00cU57SL7zk3O3KjtnK2uwv8o+wE+v6lx9l1uj9b9/bZ9jxhlI5t8KuPf/faBwdo5qqX7euWnH8kzTv3cGrV1YHzRN/dsIOu/OFb9u9VnY++vnfwO1U3c7zk1El0zY/eJq6fbXNfV85mLqer17Tvv//ZnSP8Ug0bJw9V54R2fbBs9fvLz5hi++udPX2DDNTvgvi1Gpuqabuyq7PHHOZH1V/PP+GgEd8rH3N/XfHYNt++wWU4ferX7/m54o/qG87+8es3uwafH2WT8v0XZ0/x7NvOvqbat+6F3cP6kVf/c/bPcu1XfF/f3kO/fat7RD92tpPLvPZf3w7EyG98cD9jQcdXL99yHfy937ilfBHU5mrHl3LPiGqXV/91Pq+VxtKgvCBmgvYkXAcC1ROAmKmeGe4AgZoIBBUzfhNRNflwT9jUH+O9PSYddXDriIktG8tleokWv+/5Hvek10/0OGHwPYdPaPGcQH90+jhP29ww/YSFs51qIupXn7NMnkh+/exD6e9+9cGgYFOTSzV5Ha2YcbfBzy6nMGJh45woBZmIO+vhsi46qXR+y4vb9g+bhKs+xOLTXW4Qv1aahJZ7AMr5RNXNAsw94ec63WKGr//UcR2Dk+Fyfgr6UHrVo/yyo7tIX1731jCxrAT9gvOO8Ozbznq9xIy7DXy9agf/rMS5l13Oa1nMqBcVzu/9hLWXMHf3Hy/xyALjoHZ9hKisxNfLt04h5rSdy1LPID8HD27c5fmSIOhzxXV7jS/lnhGnDW4R6/aZn28qMXH+HmKmGlq4FgSqIwAxUx0vXA0CNRMIKma83hyrSr0mijwpuPoTh9A3139AL09PngAAIABJREFU15192IhJIt/bbDHj98baC2ZQYcGToCBixjl5VxEdrtf5Pf+3V8RFiY1KkZmgky4/35bzuV+H43ve6+y3f60m3Kp9yt8cAZk6PjNMRAYRM24+Tm6VHgA/nzj9etYx4+1inNER94RRvYUPGkGrZJf6vZe4ZZYvbdtPH50+nh54bpctet0T8SB9zd3PvepSdrifyaiImXICxI9xOd/6CY2gz5+zTj8f+I0v5Z6RSu1U97IIg5gJ+nThOhBoDgGImeZwR60pJBBEzJSLlPghc/6B95sUp1nM+EW0nDyDCii/ZWZBxYzXW/paHgWnP/l+d+RJ/f6B53bSX/7BJOKonTOa5Vw+6BdxUyL5+n/fYi+PC/rxm3A6yzv7+I7BpXvOCJWKFFx6+mR72aRfFLJS1KGcrW5fM6uVFx5J8x95l24893BbIPIE1v3M1CJmgtyjbI2KmFFC1ivC6se1nG+9hIZbSASNuFUjZio9I9X0b4iZoE8/rgOB5hCAmGkOd9SaQgJBxEw1f2AZYdA3weXEjF8uDJev1oOXy5lxrhlvxjIzrxwdtt05EXavtXevc2+UmGG73LlQ7tyJII+GO9rkFrHuiRznC7Fo4ByhoJGZWoR1uYmw00avt+JqwshLfv7qtMmDfc8ZFSqXn+POcyjH0dlP+Zm7bNZke3kZiyglqM45foId8VRizt2HnOWrvuZ+Hp1v9yv5NUiOWzXLzLxyzty5W17LzNjOaifvlXzr7p/u5ZZB/VrN+FLpGammjeV848xTLOdjLDOr9ATg9yBQOwGImdrZ4U4QqIpArWLG/Ye03ITEb+lEIyMzXuKimolmuYmNe+JfzZtv5SynmFATkUaKGWWHu51Bc2a8bHVPDr2iCiqngBPZg0Rm6ilmvMpyTzad/fwnr+6hc2Z00NNb9gXKE6nqQXRN1jkXhj8qGqOiNMzJGZ0I0tcqiRmvvq36YJQiM9VM9IP41r3czi1uqonMBBlfgjwjfjlazvLVSw+nyK1myaWzX0LMVPuU4noQCE4AYiY4K1wJAqMiEETMVJpAeu2sVekNLBvdSDHjnAC6E+2DAPSaiPgtzwoywfSrM8guZeotdb1yZvxs8cvr8bo+yFtit7+dmyfwMqogYqbaKKGy1csnQaIa7uWAXssDg056K/UztbTs9iffp7/95KGDeTJ8H0+0n9rcTZzXo5ac8fdB+ppbzJS7x/3iIUpiJkhbnf6uFB11ttVrB8WgfnXb5Te+BHlGKvVv5zMJMVPpicLvQaC5BCBmmssftaeIQBAxoyZT/K8zOVphck7As1PaR+RK8HXOXYLU1sPNEjNKDHjlP/i53m9iw5NM925tlSZdlXZTcuaa+C0JqleicrlNENyRlXKPRblNBNQuVCxw3bkwql909ZjUU7QGt9wOsjV3NW+j3T7x84FTYHFf95rMu30edNIbZFjhsvccMCk3tX1wKZkSLRPH6DR9YtuwLcsr9TX10oCXbaqd2sptABBVMVMpMd7JNqhv1TjAy9re/rCPjp7YOmy3tKB+9fKBl+gN8oywTbzph3uXOKdIU6IfYibIE4VrQKB5BCBmmsceNaeMQFAxoya97jNC1GS0t2jZu1M5lwu5J5vlciic59SUiwQFza1wutFvwsf2zJjS7nvehNcEyZ3krbg4zygJMsH02wDAax0/1+E8E0PVqZY7jWbSpSbKzq2G+btqJo/lBJFz4mzs7PHciltFSJy5UF5ihpm5WQR9XL3enrs3KPAS57wpgDuHQ7Hh63kLYzUBHc0GAM7J6qlHjKUPuvs9z5zZ0d0/4kyjSknxXv7xiyqq5Y5RW2ZWzdbM5cSa19JHlZ9X68YOQcYX7vtOQenst257K0V2sMws6FOP60CguQQgZprLH7WniEBQMaOQuBPF+Xt3orE7p8A5SXRORusRmSm3UYCakPlNNtSk1OuAR3cXKCcY1ORj084eewJabvmSM0/H69BCr8TdoIdrei3tc2844Dfx9VoCEzRfhtvrd16PM9Jx6+PveYoZtzjw29ihmhwnt//cfaDc5NgpFjfv7h0hZrhs5RMWF3f86n17+3Ev/nxtNZspeEUwneLSuQOcEqJ+y6kUL77OayLt1a/cm1DUe5lZJUZ+S7GqYRjUt2rjAr/+Wy5PzunXIOPLlg/7aPpE7/O23NFAp7+dvNyHxpZbthb0gFnkzKTojz2a2nACEDMNR44K00qgWjGTVk5oNwiAAAgkjQDETNI8ivZEiQDETJS8AVsSTQBiJtHuReNAAARAwJcAxAw6BwiERwBiJjy2KBkEhhGAmBnKD/FbAsPAgi65SmL3qnTWiHtpUiMYeC2RctY7miVp9ba/0nKltPevanl7LXV1lhH0jJVq603i9RAzSfQq2hQVAhAzUfEE7Eg8AYiZxLsYDQQBEAABTwIQM+gYIBAeAYiZ8NiiZBBAZAZ9AARAAARAgCBm0AlAIDwCEDPhsUXJIAAxgz4AAiAAAiAAMYM+AAIhEoCYCREuigYBJwEsM0N/AAEQAIF0EkBkJp1+R6sbQwBipjGcUQsIEMQMOgEIgAAIpJMAxEw6/Y5WN4YAxExjOKMWEICYQR+INAH3AaTuwxPVTmGvb+8hdQhioxvENvBOePc/u7PRVaM+EBgVAYiZUeHDzSBQlgDEDDoICDSIACIzDQKNaqomwELmhEPa6QsPvklPGJ32/e7veIvmy2ZNpp6iRdf+69uD11Vd2ShuYJs27+6lK3/41ihKwa0g0HgCEDONZ44a00MAYiY9vkZLm0wAYqbJDkD1ngSUSLn7Nx/QLedPo442nTgq815nPx0+oYWe2txtiwc+c4R/Pm3aWHrktb20+NF3SZ1B01u0aEd3kR54bpddx7xzDyfnd6dOG0vnzOigzh6TntjUSZ8/dTK16sKuZ+aqlz3LOeKgFrrizKl2eXyWzYMbd9Glp0+mtoxGqx5/jz5+7Hg6NzuB+kxp/7exs4dWXngkTR3fYkduIHjQ4aNEAGImSt6ALUkjADGTNI+iPZElADETWdek2jAV7TjrmPG2WGHR8O1LjrWFCQuKYya10fee2WkLhfmPvEtnH99BfC2LkOeun2kLGxYSay+aTveu304XnnjQiO9YePCHl6epe1gMqbqVQHKWs+dAkdb+6n37PnUPl8ORmW17++0o0ZfXvUXnHD/B/nndC7vp8jMm2zZiGVqqu3QkGw8xE0m3wKiEEICYSYgj0YzoE4CYib6P0mghC4Xnt+63BYpaPvaba0+kbz21nb44e4otHvijoiQqUnLPb7bTZ086mG59/D1bPHA5r77fQ7OOGjvsOxY7SoSwUPr+pcfZeS/qs3HrfmrPiBH3sLC55y+OtiNFKvqiyuF7WWSxOOI8mr//86Pp12920aeO62jaErg09h20OTgBiJngrHAlCFRLAGKmWmK4HgRqJAAxUyM43BYqASVmzj/hIDuqkZ3Sbkc6fvLqh3TZrCn0nd/toEtOnWRHajiawh93ROXJNzoHozkqMuP8TokQXvrljMyohqnv1D3r3+oeFEVb9/YNll0pMsPiqln5PKE6CYXHngDETOxdiAZEmADETISdA9OSRQBiJln+TEprvvP5Y+2oDOfIcA4K56dwXgp/OBeFP2pJl9ocQOXZrN/cZee/8Kerx6Rvrt9u/8w5M87vnGJG5dlwzgx/vrthh71szOselRPz7p4++uWm0sYEl58xxTdn5sZzD4eYSUrHTFg7IGYS5lA0J1IEIGYi5Q4Yk2QCEDNJ9m5826a2XN7bY9p5MOoTZCtmd0SFozdekRkV0fGj5FVOpXviSxyWp5FAScxo+adp1i1pbD/aDAJhEoCYCZMuygYBBwGIGXSHKBPg3cpOPHTMoIkqT6WcqHBGWdw7kzl3K6vUbq9yKt2D34NAnAhAzMTJW7A1bgQgZuLmMdgbWwJnZJ7tevFvzPH942LbBBgOAiAAAiBQA4FjfyEOTH1FzH2aZt9dw+24BQRAoAwBiBl0DxBoEIHT25/5xTufkuftOEk2qEZUAwIgAAIgEAUCp39b62rp1j69gWY9FwV7YAMIJIkAxEySvIm2RJrAmfT8BX0Tij94/gprQqQNhXEgAAIgAAJ1I3DIS4KO/K14duOB2bPrVigKAgEQGCQAMYPOAAINJHBG6zPf7zyKLip8FoKmgdhRFQiAAAg0hcDETYJyD2smSXEmojJNcQEqTQEBiJkUOBlNjBYBFjRmO1307plywp5jJSGHJlr+gTUgAAIgMBoCwiQat13Q1FfowOTfa716P31+A53x89GUiXtBAAT8CUDMoHeAQBMI8JKz/vbidVpRfEwv0vgmmIAqQaChBLJnbaa3njuSin2ZhtaLykCg4QQEmcU2ekfvkd8TZK7dQB8rHZKEDwiAQCgEIGZCwYpCQQAEQAAEnARWL16+VWr9s+fl89tABgRAAARAAATqRQBipl4kUQ4IgAAIgIAvAYgZdA4QAAEQAIEwCEDMhEEVZYIACIAACAwjADGDDgECIAACIBAGAYiZMKiiTBAAARAAAYgZ9AEQAAEQAIHQCUDMhI4YFYAACIAACCAygz4AAiAAAiAQBgGImTCookwQAAEQAAFEZtAHQAAEQAAEQicAMRM6YlQAAiAAAiCAyAz6AAiAAAiAQBgEIGbCoIoyQQAEQAAEEJlBHwABEAABEAidAMRM6IhRAQiAAAiAACIz6AMgAAIgAAJhEICYCYMqygQBEAABEEBkBn0ABEAABEAgdAIQM6EjRgUgAAIgAAKIzKAPgAAIgAAIhEEAYiYMqigTBEAABEAAkRn0ARAAARAAgdAJQMyEjhgVgAAIgAAIIDKDPgACIAACIBAGAYiZMKiiTBAAARAAAURm0AdAAARAAARCJwAxEzpiVAACIAACIIDIDPoACIAACIBAGAQgZsKgijJBAARAAAQQmUEfAAEQAAEQCJ0AxEzoiFEBCIAACIAAIjPoAyAAAiAAAmEQgJgJgyrKBAEQAAEQQGQGfQAEQAAEQCB0AhAzoSNGBSAAAiAAAojMoA+AAAiAAAiEQQBiJgyqKBMEQAAEQACRGfQBEAABEACB0AlAzISOGBWAAAiAAAggMoM+AAIgAAIgEAYBiJkwqKJMEAABEAABRGbQB0AABEAABEInADETOmJUAAIgAAIggMgM+gAIgAAIgEAYBCBmwqCKMkEABEAABBCZQR8AARAAARAInQDETOiIUQEIgAAIgAAiM+gDIAACIAACYRCAmAmDKsoEARAAARBAZAZ9AARAAARAIHQCEDOhI0YFIAACIAACiMygD4AACIAACIRBAGImDKooEwRAAARAAJEZ9AEQAAEQAIHQCUDMhI4YFYAACIAACCAygz4AAiAAAiAQBgGImTCookwQAAEQAAFEZtAHQAAEQAAEQicAMRM6YlQAAiAAAukksPa6tWOuv+P6A9x6RGbS2QfQahAAARAImwDETNiEUT4IgAAIpJDArTctuVQj7Z9Jo3ymc9cdxY4pm6TWP1uYLRcIIW8nkt+as+SmuSlEgyaDAAiAAAjUkQDETB1hoigQAAEQAIEhAqsXL9tJRGMFkUkkdEmyU0p5kCDRYur6CfPz8zeBFwiAAAiAAAiMhgDEzGjo4V4QAAEQAAFfAqsXL72apLidBI1xXNQriP5lzpJFVwIdCIAACIAACIyWAMTMaAnifhAAARAAgTKCZtn7RHSoukCQ6C9q2omIyqDTgAAIgAAI1IMAxEw9KKIMEAABEAABTwKu6AyiMugnIAACIAACdSUAMVNXnCgMBEAABEDATWD1YhWdEf0mojLoICAAAiAAAnUkADFTR5goCgRAAARAYCQBOzpD2h0k5QNzly66AoxAAARAAARAoF4EIGbqRRLlgAAIgAAI+BJYdfPSH1jCvGl+Po8dzNBPQAAEQAAE6kYAYqZuKFEQCFQmcCY9d1VxjHW11itO1CxqrXwHrgABEAABEIgDAUujPqtNvpY5oN27gWbdFwebYSMIJIEAxEwSvIg2RJ7AJ2jjET1jzEcOTBbHbJtlHdx1pCQTUibyfoOBIAACIBCUgN5H1PGuoCOe0/aM2SU3tx/QL1xPp28Lej+uAwEQqI0AxExt3HAXCFRFYFb7M79//3SZ23qmrOo+XAwCIAACIBA/AtM2CDpsoyg81zP7I/GzHhaDQLwIQMzEy1+wNoYEPqo9s3x3jr626U+tcTE0HyaDAAiAAAjUQGDGf2r7JhXozqet2QtruB23gAAIBCQAMRMQFC4DgVoJzM480/nyX1sdPRNrLQH3gQAIgAAIxI1A+4dEJ/+L1vVMcfaEuNkOe0EgTgQgZuLkLdgaOwJn0bMn93TI3zx/pYU/ZrHzHgwGARAAgdEROO07Wmd7l/jkU3TGy6MrCXeDAAj4EYCYQd8AgRAJfIyeOaf7EPHjVy4zDw6xGhQNAiAAAiAQQQInPaDvGb9dfu53NPvJCJoHk0AgEQQgZhLhRjQiqgQgZqLqGdgFAiAAAuETgJgJnzFqAAGIGfQBEAiRAMRMiHBHUfQfZSfQ9y89jqYdNLQ/9msfHKCZq0orQbx+76zuuxt20JU/fIu+8/lj6fwTDqIvPPgmPWF0jrDosatPoHOz3isMnfXxjVzWFWdOHSyjq9eka370Nt3/7M6K9jivrYTFq22PG5103r2vV7TfXY8q69HX99o83J/Lz5hCay+aTveu306LH33X1zRVzoR2fbDN6uJyvlB+cBbsZu7Fxs8vfaakVY+/N8JWbsc9f3E0dfaYg75ecv6RNO/cw6lVF57tqtSf3P5XdXS06SPKc1/LF7w672Q68dAxg9du3dvn2w8r9Qn8PjwCEDPhsUXJIKAIQMygL4BAiAQgZkKEW2PRanL8+vaewQm8+o6LZGHCHxY7fpN0VXUQMXP4hJZBkeRnMk9Mjzq4ddhEnifcnzquw3Nyzb8LUq67PjUB//WbXYNtV5Pod/b0jRBzbnGhBJcSEfUSM1zuRSeVdsh4cdv+YcLKrw6+h21X4sNPELlt5jr8+PH3H50+boSg4u9POKSdmMe6F3aPEG7lOCjmm3b2DOsHbp/7CT/ln6e37BvkwvfyR4lvJW7cfajGRwS31ZEAxEwdYaIoEPAhADGDrgECIRKAmAkRbo1F8+Ty62cfSn/3qw+GvYF3fv/kG50NEzM82b7k1EkjJtBqguqetJabjJdD4iXi1PVuJmry7hVxctrL0YByoi9oZIYn5+919tvmsGhw1usnFNxlB7WZI11+YkYJB6dgcdZ/1jHjR4gI/sLPxnLM3f4tx8rZNo4mcpTILaq8bK/xEcFtdSQAMVNHmCgKBCBm0AdAoPEEIGYaz7xSjV6RCPc9lSIO6vp6RGa83rJXakMtkRk/EefXdmfkynmNk82DG3eNWsw4J/Fcj1tolovMsLjg6EQl0eAuoxoxw9yu/sQhdP2/b6Gzj+/wFJ5+NlZi7iyb2+63JM8pZvg6FpD88VveWKn/4PeNIwAx0zjWqCm9BBCZSa/v0fIGEICYaQDkGqpw5zu4cyXK5Wk4czCCiBm/nBleqqXEgJ9w8GtaLWKmXATIWU+QN/wqkrLisW2jFjPuiIpb3JXzhcoT8YtWONulbObcoGqWmTnt8RNN5QSXX9SNbXMKOWNnj6eY8Vpm5pVf45U/VMOjgVvqTABips5AURwIeBCAmEG3AIEQCUDMhAi3TkU7E+/V5Fi9/Q47Z6ZSRCHpYsZLBLhFV7moByfg87KxX73R5bn0qpyY8RKZ7s0CvJZ+eS1nq5eY4eVjXhsA+G3QwO1zCvNqNoKo0+ODYioQgJhBFwGB8AlAzITPGDWkmADETHyc74xIVFo+pVoVJDJTKVE/zcvM3Du4OXuLmsCXW/Kn2F37r2/bEaIgS+N41zV3ZMZrYwS2pdxudM5ISFjLzLw2hvB7omoVxvF5QuNpKcRMPP0Gq+NFAGImXv6CtTEjADETPYf5JYo7J4OVlk/VU8yUW/7lZ2sty8zKTXbdS8uCJtOPdgMAPyHH3x/UrlfcWc69dMy9eYDTT87lXl78/HZqcwskxXFvjzli9zd3JK+SwHC23ysK5N5lj7f/rmXDiOg9hemxCGImPb5GS5tHAGKmeexRcwoIQMxEz8l+GwBUs0tXPcUMl+W3NTMvhfI7S6VSxMeLfLmtmZ1nqATd5rjSRgnldugql5sTZGc5d3J9UJtVxMWLH/thxpR2e7tn/njtesffB10Kx9eOdmtm9/1eAkfV42dv9J7C9FgEMZMeX6OlzSMAMdM89qg5BQQgZqLpZK+kcuehg+WSzrlF6hBDv2VSqqwF5x3he2imO78hyGGPimYtkRl1r1fy+GgPzXQePqrqYRGmclnceSC84QIzmjIu47kltTP6oZaQ+dXhPqwzCEc/fk5BxOKOP147hrmT8iuJOq/+5HVopt9uZqqfOf3kPjQT+TLRHGsgZqLpF1iVLAIQM8nyJ1oTMQIQMxFzCMwBARAAgQYSgJhpIGxUlVoCEDOpdT0a3ggCEDONoIw6QAAEQCCaBCBmoukXWJUsAhAzyfInWhMxAhAzEXNIQs2ptCyOm41zSBLqfDQr0gQgZiLtHhiXEAIQMwlxJJoRTQIQM9H0C6wCARAAgUYQgJhpBGXUkXYCEDNp7wFof6gEIGZCxYvCQQAEQCDSBCBmIu0eGJcQAhAzCXEkmhFNAhAz0fQLrAIBEACBRhCAmGkEZdSRdgIQM2nvAWh/qAQgZkLFi8JBAARAINIEIGYi7R4YlxACEDMJcSSaEU0CEDPR9AusAgEQAIFGEICYaQRl1JF2AhAzae8BaH+oBCBmQsWLwutAwH3IpPswR7VT2uvbe+i8e1+vQ43VF8E28KGZ9z+7s/qbcQcINJEAxEwT4aPq1BCAmEmNq9HQZhCAmGkGddQZlAALmRMOaR92yr37uyXnH0mXzZpMPUWLrv3Xt+kJozNo8XW7jm3avLuXrvzhW3UrEwWBQCMIQMw0gjLqSDsBiJm09wC0P1QCEDOh4kXhoyCgRMrdv/mAbjl/GnW06cRRmfc6++nwCS301OZuWzy8Ou9k++fTpo2lR17bS4sffZf43nnnHk69RYt2dBfpged22Za4vzt12lg6Z0YHdfaY9MSmTvr8qZOpVRd2PTNXvexZzhEHtdAVZ061y+vqNenBjbvo0tMnU1tGo1WPv0cfP3Y8nZudQH2mtP/b2NlDKy88kqaOb7EjNxA8o+gUuLXuBCBm6o4UBYLACAIQM+gUIBAiAYiZEOGi6FERUNGOs44Zb4sVFg3fvuRYW5iwoDhmUht975mdtlCY/8i7dPbxHcTXsgh57vqZtrBhIbH2oul07/rtdOGJB434joUHf3h5mrqHxZCqWwkkZzl7DhRp7a/et+9T93A5HJnZtrffjhJ9ed1bdM7xE+yf172wmy4/Y7JtI5ahjapL4OYQCEDMhAAVRYKAiwDEDLoECIRIAGImRLgoelQEWCg8v3W/LVDU8rHfXHsifeup7fTF2VNs8cAfFSXhnzlScs9vttNnTzqYbn38PVs8cDmvvt9Ds44aO+w7FjtKhLBQ+v6lx9l5L+qzcet+as+IEfewsLnnL462I0Uq+qLK4XtZZLE44jyav//zo+nXb3bRp47raNoSuFE5ATcnngDETOJdjAZGgADETAScABOSSwBiJrm+jXvLlJg5/4SD7KhGdkq7Hen4yasf0mWzptB3freDLjl1kh2p4WgKf9wRlSff6ByM5qjIjPM7JUJ46ZczMqPYqe/UPevf6h4URVv39g2WXSkyw+KqWfk8ce8HsD9cAhAz4fJF6SDABCBm0A9AIEQCEDMhwkXRoyLwnc8fa0dlOEeGc1A46sJ5KfzhXBT+qCVdKulf5dms39xl57/wp6vHpG+u327/zDkzzu+cYkbl2XDODH++u2GHvWzM6x6VE/Punj765abShgOXnzHFN2fmxnMPh5gZVW/AzWERgJgJiyzKBYEhAhAz6A0gECIBiJkQ4aLoURFQWy7v7THtPBj1CbIVszuiwtEbr8iMiuj4GepVTqV7RtVo3AwCDSYAMdNg4KgulQQgZlLpdjS6UQQgZhpFGvXUSoB3Kzvx0DGDt6s8lXKiwhllce9M5tytrJJNXuVUuge/B4E4EYCYiZO3YGtcCUDMxNVzsDsWBM6iZ0/u6ZC/ef5Ka0IsDIaRIAACIAACdSNw2ne0zvYu8cmn6Iyh8GfdSkdBIAACTABiBv0ABEImMDvzTOfLf2119EwMuSIUDwIgAAIgEBkC7R8SnfwvWtczxdl4mRUZr8CQJBKAmEmiV9GmSBH4qPbM8t05+tqmP7XGRcowGAMCIAACIBAagRn/qe2bVKA7n7ZmLwytEhQMAiCAyAz6AAg0gsCs9md+//7pMrf1TNmI6lAHCIAACIBAEwlM2yDosI2i8FzP7I800QxUDQKpIIDITCrcjEY2m8AnaOMRPWPMRw5MFsdsm2Ud3HWkJHPo/MBmm4f6QQAEQAAERklA7yPqeFfQEc9pe8bskpvbD+gXrqfTt42yWNwOAiBQgQDEDLoICDSQwJn03FXFMdbVWq84UbMIcqaB7FFVuATGHnSAxk/eT+Mn7bP/7eB/J+2nA11tNKajl6QUtH3zJOrcPp66d4+j7l1jaf/eoV3UwrUOpYNA+AQsjfqsNvla5oB27waadV/4NaIGEAABJgAxg34AAiAAAiAQiMBdK1ZM7TlQzBHpWdJkVloypwmRlURZIrGbSBpEoiDJMjQSBaFpRjf1bRpPrTMsy1pPwrqbpH4okcwRiSyRnCSIDEtKQ2iiQJYwiEyjfUym8NUFC3YEMgoXgQAIgAAIpJoAxEyq3Y/GgwAIgMBwAvfk8+N7zLasKcwcScqSJrIkWXywYLE/hiQyhKQCCTJ0aRW6dMvI5/Pd5ViuXrx8q9T6Z8/L5weX3eTz+fEdppY1hWbXJQXlRKmewboZ5d+qAAAgAElEQVRIsMiRRqkuvdCu9xrXVKgLPgUBEAABEEgPAYiZ9PgaLQUBEAABm8DFF1+snznztKy0rKxFMidIywohs5JFixSThaCCM1qi67LQ26sZC1bWHi3xEjPl3LFi/oqpbW1W1jRFblgUSFKOhNwlhChIKQxnFGjDq88bDz30kAk3gwAIgAAIpIcAxEx6fI2WggAIpIzAbQuXHS1beDmYnpXCsqMfGvGyMDvSYgiigiU4yqIVhGYapmwp3HjLjVvCwFStmClnw6033zpdF/051S6NozpEdvRIkChYvNzN0S7RL4wbli96O4x2oUwQAAEQAIHmEoCYaS5/1A4CIAACoyJQYx5LIZ/PW6OquMqb6ylm/KrO5/PaeGrNOSNOyM+p0lG4HARAAARiRgBiJmYOg7kgAALpI1A+j0WKUtJ99XksjSTZCDFTrj3++Tn2ZgR8AJSB/JxG9gjUBQIgAAL1IQAxUx+OKAUEQAAERkXAK4+FhCwl3w/ksZCUhnTs+lUsZgqjyWMZlcFV3txsMVPOXM7PyWSGdmkTlsxRaZe2HJHcZYsc5OdU6XFcDgIgAAKNIQAx0xjOqAUEQAAEbALV5rEkJd8jymKmXNd0+wv5OXiQQQAEQCBaBCBmouUPWAMCIJAAArXksSR9J664ihm/7ugZScP5OQl4etEEEACBuBGAmImbx2AvCIBAJAiUcjD8zmOJRx5LI0EmTcyUY1dLfk6X3lvxrJ5G+gt1gQAIgEBcCEDMxMVTsBMEQKDhBMrmsZCYzFsbcy6FJSx7e2OcXu/vojSJmXId1Rm14+2yNallOTcK+TkNf7xRIQiAQEIIQMwkxJFoBgiAQO0EhuVFSCtHovx5LEnJY6mdWPV3QsxUZhYoP0eSIUTpXCD0w8pMcQUIgEDyCUDMJN/HaCEIgAARIY+lud0AYqZ2/sjPqZ0d7gQBEEg+AYiZ5PsYLQSB1BAYmccis1KKnCDKEnnlseiFdr3XuCaf704NpCY1FGImHPDuM4ikINXfHefnyAJZwiBBhi71AvJzwvEFSgUBEGgOAYiZ5nBHrSAAAjUSqJTHwocfCiEKlkQeS42IQ7kNYiYUrGULHZGfI7Ss5HOLSuIe5+c03iWoEQRAIAQCEDMhQEWRIAACoycwlD9AdnSllMdCA4nSZHDyvSX41HvkD4yedvglQMyEz7iaGsrn51DBIjLIzs+RBaER8nOqgYtrQQAEGkoAYqahuFEZCICAk4D75HVpyZxWOnk9SyR2E0mDSBQkWQZZZOgZvTD91eeNSx56yATJeBGAmImHv9ZdfLG+ZeZpWbNo5kijrCCNozi8ZI3/nSSIDEtKQ2iitHSNTKNYzBQWrFywIx4thJUgAAJJIwAxkzSPoj0gEDEC7jM3SBuZx8ITJCmpwGv6qSiMca39BeSxRMyRozQHYmaUACNwO+fn7OtryVHm/7f3JmByVeed93tu9aYdkAQGsQlUNUQsISBZ2H6IjSFjBnusLE/gC/5sfzFOHMZjYhFLQkKSL1rQAgjHNmaI7UnAMUzkSTJJjE2MwXZsjMWOWAZXyYhFYtECqFtIvdS953vOrb6t6lJV1+3uulV3+ZUfP7Sq7j3nfX/v6dP3X+95z9FZ0ZJVSnKlLx7ELF3T2lviWV6f4+Z7Mi7n50QgdpgAgSQTQMwkObr4BoEmERhVHYtl5cVxCirjFhbb9u4mmUg3LSaAmGlxAELu/ibbPlY7VlYymax23ZxFfU7IxGkeAhDwCSBmGAsQgEBgAkHrWMySMCuj85yDERht4i9EzCQ+xDUd9OcN11He0jVLD9W+Zb3aN+pz0js48BwCDSCAmGkARJqAQJIIBK9jkYK47mAdy5zC5d+7nDqWJA2EBvuCmGkw0AQ0t+WPt2ReOXP7YH2OZYQN9TkJiCsuQKDZBBAzzSZOfxCIAIGR61hEiUh+WB2LpQqThDqWCIQutiYgZmIbupYY7tXnSHtOXOpzWhIAOoVAjAggZmIULEyFwGgI2LZtTZaOnHbdrCs65+1KpHVWlLcz0fRh57F4dSyqoDL91LGMBjLXBiaAmAmMigvrECjV53RkJaOr1+dolRelCmYXREtUXllW4YD0523bdoELAQgkjwBiJnkxxaOUEdjw5Q0nZ9RATrvVz2MRrQva20aVOpaUDY1IuYuYiVQ4EmtMZX2OcnVOlNlW2mwzfWR9jqPb89fdcN0riQWCYxBIAQHETAqCjIvxJ2DqWDo73azjFdCabyNLf6BLa8xL57EoUaVCWr+OZe6cwuWXU8cS/+gnwwPETDLiGFcvtmzZknnl+cP1OaUDeA+fn2O2lTZf/Pjn52QyOt/XZxU4PyeuEcfuNBFAzKQp2vgaaQJ+HYtrma1NdUm0aJUrCZayOhZLvCyLWKqgD+4vLN20qSfSjmEcBEQEMcMwiCqBjUuWTFETp2W9+hxzUKhb+/wcI3Ys1ylwfk5Uo4ldaSSAmElj1PG5ZQTK61jMH03tqpypY1HK26p0Rnkdi/ezzuSpY2lZuOi4gQQQMw2ESVNNI+DX52jlmINBs0ecnzNYn6Msc1ioFKjPaVpo6AgCQwQQMwwGCIRAwKtjcQZy2jJZFctbzmCWNYgo8/N2fzmDazIsShUyjs5/ad2Kl0MwhSYhEAkCiJlIhAEjGkjA1Oc4mdIXUpb35dSR9Tlm+a+IW1CuFJwM9TkNxE9TEEDMMAYgMF4ClXUsMngQXHkdi2gpaKXyYrkFKUph9jn/KU8dy3jJc38cCSBm4hg1bB4LAVOfs2Pbr3PSJllxrazSRuSUvswS0ceY+hyzGYEoU+OoCtTnjIUy90DgMAEyM4wGCIxAoFodi2hr8GC3w3UsrqiCEp0v1bG0FZZuWkodCyMLAmUEEDMMBwiIbFyycYqaWPTqc7SonCXmv15dpFnGpsVkcpTriRzqcxgxEAhGADETjBNXJZhA0DoW0W7B7BZGHUuCBwOuNZTA5kWbJ1x767WHTKOImYaipbEEEiivz/GWJSsrq01Wp7St9F6tS8uSqc9JYPBxaVwEEDPjwsfNcSJQrY7FK+gcqmMxxZuSH6pj6dD5L62gjiVOMcbW6BDYsHL1lZZYfyeW2G3d+24tTpmxXVsD85XTfplS+hYRfcfi1SuXRMdiLIFAdAncvHbtKU5/eX2OeEvXlKg5ruiCt2EM9TnRDSCWhUoAMRMqXhpvNgHqWJpNnP4gUJvAplVr94rIRCXiiKiMFt2ttZ6mRLU7mcwZy+xl2+EHAQiMnQD1OWNnx53JIYCYSU4sU+NJzTqWUpGl+V9emXS8qIKmjiU14wJHo0dg06o1V4tWt4iSCWXW9SmRv1+8esVno2cxFkEgOQTK63OUqJyY+hyzEYGWnGjRYjanoT4nOQFPsSeImRQHv5Gub1q59psHM8WrbdsuNqJdU8fSIR25dtf1DjGrdR6LX8eiHVXItBfzi217dyP6pw0IQKAxBDatWvuGiBznt6ZEDRQtay5ZmcbwpRUIjIWAqc9xBtpyKqOzQepzBiyr0C/9edu23bH0V3mPbdttE3X77UtuuP7PGtEebaSbAGIm3fEft/c32Tcdq3Xft0XLf1aiVy1evXLjaBoNUsciShf04MFkrjVQWGbbL42mD66FAARaR6AiO0NWpnWhoGcIBCKw3rZPtdz2bOlAZ50TrbKNrs+5adW6pVq0LUp+rFTnVYvtxXwRGSg6XFSNAGKGcTFmAptWrf1jEfmmaOkSJW1dM6ZOuuaaa/oqGxyxjkXL22ZZWGm/fSnojJWXolM41OYUGpXlGbOD3AgBCDSEwOHsjBpwyMo0hCmNQKDZBEw2ZUIxk5W2TFY5bs6smjDnq3nL1pQcPZrzc7761a929u7tfldEFUV0r4j82ZLVK77XbJ/oLxkEEDPJiGNTvdBaq01fXneHJfInWmSy6VyJ/Lul3WWulSmdgmzprHcei1/HInpwpxVVcLRTsDKZPOexNDVsdAaBlhHwsjNi3Spaf3fJmhVXtcwQOoYABEIh4NfnuI6Ty6iMOYstq8VsK62M2Klan+OqzHot+iODzxAHXJF7ltxw/eeUUjoUI2k0sQQQM4kNbTiObVq19k9EZK0WOVGJdAz2Ys6RaBeR5/3tIV1xC6U6ls486eNwYkGrEIgTgY1fXnOPq5yVy2ybHcziFDhshcA4CZjl6M5A32B9jjUocsw5OjJXXBnwNwjRIv1KZKclsuJLq1fcM85uuT1FBEYvZq75P5dJe9sicZwLxHW9b+V5pYfAn0zcKWd3dJtMjGTMYcWDL0eU3Nw9R/a5vr5JD5NEe2pZByST+ZUMFG+Vr/7+D8bsK/PGmNFxIwRiR4B5I3Yha7bB061++dLU7Uc8R2gt8kxxqtzz7onNNon+Wk1gHPPG6MTMX/3bXWK1LZTTZk+VY6aLdHa22nX6bwGB9xzaJ2fuf1Hm7Xtepg68K5bWktGOvDT5ePlGzpTR8EoMgb4+kbf2iby4o1vc4r/ILf/1U6P2jXlj1Mi4AQKxJsC8EevwNcP4/5b/npx64HVxVEZcpaS7fZI8Nn2uPDftNHljwvRmmEAfUSMwjnkjuJgxDyTTjl4o55wzNWr+Y0/rCBzb+7bM3f+izN/3vEzv2y//cezvyA9mfaB1BtFzeAS2beuW/W+PTtAwb4QXD1qGQBwIMG/EIUpNtfGyXQ/J7+5+QvZ1HiWPTp8rz087TXZ3Hd1UG+gs4gRGOW8EEzNmiUhn5z3ygQ8gZCIe/1aad3rPTvnNFFLDrYxB6H0/9FC39PX9SaAlZ8wboYeDDiAQCwLMG7EIUzONnNOzU7bzvNBM5PHraxTzRjAx81ffv1/mzLlEjj8hfjCwGAIQaByB118T2b79x3LLx36vbqPMG3URcQEEUkGAeSMVYcZJCDSUwCjmjWBi5ov/0iPve/9kamQaGiYag0D8CJg1rQ//8oB8ZeGUusYzb9RFxAUQSAUB5o1UhBknIdBQAqOYN4KJmWv+WcuHL26ojTQGAQjElMCDD4h89Q/MhnYjv5g36hHicwikhwDzRnpijacQaBSBgPMGYqZRwGkHAmkhEHByEcRMWkYEfkKgPgHmjfqMuAICEBhOIOC8MSox8+EZGbnr/Akyq6v6l7I9RS2f39Ynnz6pXS6emTkiJP2uyMZCv6x6oW/YZ588qV1uO6dTuosin3r8kDy41xn63P9sy66ifPapXvnWuV1y1Snt8u2XB7x/+y/fthd6XLnklwcDDYfVZ3TK0myHdFiHLy9v1+97StuR/vq+/HRv0WNy35sl+ypf9ZhV+vHj908cxm5Xrx5i4vtey7kH9jhyY75vxBj5/VX247f5fI8rZz74biB+5ReNZPdI1/lj5juvDgxdZto6vktVtcPnWYt3tb5qjZVq47jyWtNeJfdq47gWz1pj3vdjapt4vzPG/3pjpZxVtWsrYzfS+K0W5+c+PEnmTjn8y1A+9oYNiICTiy9mgvrFvHGYcj1mzBulvzXMG8wbzBvMG7X+VlX+XeZ5I7nPG6MSM5VPuLUeOkd6/71HW1X/CJ0xxRLzYOeLFr+vWmKmclCOVswYIfPF09vlK78ZGBJXvrj5+T7HE0Sm781ndcrtOw5fU0us1BMz9R6+fftN++WCzjxgnjRBHcGslm1BH/arxci/d/+ADixogtpd7eG9XCiUP5w1Qsz4fLoHtHRm1DCmtRgZ0WLuKxfchv+cSdaw9yrHifFjtGPe9LXw+DZvOD29v7oAr9Wm3//2d4cLz8qxUmuM+L9Tj7x9uF9zr3mVC9laY09GKWaYN478bqDe72m9zyu/xGHeqP07aNgE5cm8MVDz7x7zxmHxzPNGaQYK+nvF88aRX3jzvNHY542miplKYVL5y/C+Y0rZnPIHqmpixn8I3N13+KF7tJOL+eUyAqoyE1T+/qwJVtPETC17avkVhpgx7KvFaKQ0TVC7a13nC5rLZ7UNCbZGiBkzUVx6XJv879cG5DMnDxettSbgSqamjXK7yjlUiuFaNtfiaYTC673aa7LaOKwlkOqN83JRMpIYrxznJjNa74uEIf+bLGaYN47M+PqxCPr7V/7lULUvaMbzUMK8MfxLL+aN0kqLyvmk2V+CMG8wb1Q+3/G8kdznjZaLGfNQePXsdrn22T754PTMEQ+P1cSM/5B69akd3jIDs7yr3kNe5QN5reVq5dc1KzMzWtv9h4cwHkpM2/6Ddr3lekHtrndd5YNUI8SM74O/7K58+eFImRkjqH0xXS1bUT4+yjmNRsyUjyvTXmWGsPxBtXK5XbWMYqXI8n+fzPu1MovlD8HmOrNUsvLb/ZoiNgJihnnj8Leio1laG9aXIMwbzBvly8Orzh3MG0csj+d5g+cNnjcOl5WMZ95oqpgxD1CVy8zKHxirPfTWEjNGcS/PdQ61t+uQ6z2QjeYPe706j5FqDvw1mv6DYL1lZtXqM/ylcqaNqt9kjZASqfdQMlJ/RgCOJBhG+qxy8g1id5BsTxBhYPoO8g1y5QN/5TelI9UjVMZ1pPFUzmk0y8wqv02vJZqqtTnSt76VIrfwrltVzFRbLlJtrFerH/Li3+SHEuaN4RMB88ZhHswbw1cyGDLMG6XxwbzBvFGtjrnyi5fxfnnK80apFKPVzxuhiZlqGwBU1rlUeyCvfNAbScz4QsLUeHxhW++oxUzlt+x+8bMppG9mzUyQh/1KXVNPzNSr0UmymKn0rZJvLUHk16IYsXf3zoG646lSzAQZ89X6riVQGvFQYsRmtQ0s/DFeTS+Xb4xRrWAyTDEThCHzRilqzBvDv9Ud70MJ88bhhxLmjcMbDpmVIObLU543jqwfDvLlYi2B7f/tifuXp8wb0Zg3QhMz5ctjqhVM+wO82sOL+cz/VngkMWPS2n7b//R6US6cnhlVZqbag1z5g6X5vBkbANRbhlXNzjDFTJyXmY03m1aeJRnPMrNaY36kHekqBUa1SX68y8xqFvVXGWQ1x2WTMjPMG7V3SSzPUo4mG828UX3XSeaNw8u9q/3dY94o7bLK88bwzZAaIWZ43ihlV3neKO2WfMTfs4DPG00RM2ZyrKxRqfWgVLmjVj0x44siI2T6XC3lOzRVEwH1HgLKHxZrpc3K2633y1zv8/JvJ2oVglcb5GE9lIz2296RCvvL7W7WBgC1shzlO4eMtJ12+cQ63kLeanVZtSYs8/609uG7rlUTM/WEb70NAPz7zbjziyNH8rOqvQEnl1rnzIy0JK+yRoh5o/rDN/NG6W9KozYOYd44vJthtb8tzBuHj4ww8xfPG6UZaDTPN9WOW+B5o7QRkFkJxPNGKdNuXsOOBwn4vNE0MWMMLN/m1vy7VuFzZXakvC7D36WqfJeK8hqIkZbPlIuQWhsA1HsYrBRI9X6Z633ut1ftj0Uls/LzecIQM5VCspYYrCbmyh+Oq9ntt11+rko1keuL07GeM1NLLJSLAH9TgMqleNWyHuPdmrn8fiOOa9UYVeu71kP/eLdmrry/1tirmQUKOLk0Qswwb4wsZpg3Dp83Np5lZswbh7f/r/W3hXmjVKjM88bhcwKDPt+M9MVco46CKD9CgeeNfq+GJU3PG00VM+UDzByQWfkA7D8klxco3/nqwLCAVBMz5r5aS1JGeiCvdmhmuRgaaemBabfeIZVmqZxfe1HroNHKwwvNH9UgBxfWEzP1+qvc/MDnFFQMVnINandlv7UOsaq2/NCw8mujqvn35H5XzpxiDe1wV2mjXxB6247SWQqNPjSzmgArH/OvHNJycpUzg8r/QJZP7EEezsp9qHZoZq1lkr6YL493ZQyr1ssYY5ssZpg3jpzFmDeqH5rJvBHsvB3mjcO/Uzxv1D8omeeNIw9O53lj+Dl3tcZI+ZfWYT5vjEvMBPnmnmsgAIGEERinmEkYDdyBAASCEGDeCEKJayAAgXICAecNxAzDBgIQGB2BgJNLrWVmo+uMqyEAgUQQYN5IRBhxAgJNJRBw3kikmBlpxygTBP8skbqHfDU1YtHqbKSzWHxLa55DEi1XsKbRBAJOLnETM8wb4x8ozBvjZ5jYFpg3Ehva8TrGvDFeggm+P+C8kUgxk+Cw4hoEWk8g4OQSNzHTerBYAIEEE2DeSHBwcQ0CIREIOG8gZkLiT7MQSCyBgJMLYiaxIwDHIDB6Aswbo2fGHRBIO4GA8wZiJu0DBf8hMFoCAScXxMxowXI9BBJMgHkjwcHFNQiERCDgvIGYCYk/zUIgsQQCTi6ImcSOAByDwOgJMG+Mnhl3QCDtBALOG4iZtA8U/IfAaAkEnFwQM6MFy/UQSDAB5o0EBxfXIBASgYDzBmImJP7NarZyB6bKndrKP695COLgqcZfO6dL/vWNonz8PW2yodDvHT5Z/jI7jphrqn02Wn/NAUu7DrnCjnKjJReB6wNOLoiZCMSqhgnMG9GNTWItY96IfWiZN2Ifwvg5EHDeQMzEL7RDFpuJ5fJZbfL5bX1DwqP8vewkS66e3S7XPlv63Hz2vmMycuaD7x7htS9UmiFmGimKYhy++JoecHJBzEQzxMwb0YxL4q1i3oh1iJk3Yh2++BofcN5AzMQ0xCazsX5up9yyvV/+ak6HzOpS8uK7rrxT1PJ2/2GnXjroymef6h16w8+IXHliu1x1Srv3vjkv5u6dA17WpVzMXJftkLlTLO+a53tc+cK2XvnmuV0ys1PJlDYlD+xx5JJfHvREUrW25kyy5Of7HDm+Sw1r5+G3HO96k0W6dXu/LBq0388qGdt+//g26bBkmFCLaaiSZ3bAyQUxE73QM29ELyapsYh5I7ahZt6Ibejib3jAeQMxE9NQGwFx6sSS0DAvIyqe+/AkMULhtV4tH31PxhM1lWLGXOtPTMue7/PuNaLIZG7M8jJfzJj/GgFj3jfXG2HzP18ekP82u12+u7MoP91b9MSPESsfe0+bVLZ1+QltQ9fNmmAd0c5nTmn3lqt9+qSSoDL2//j9E72fjc21MkgxDVeyzA44uSBmohd25o3oxSQ1FjFvxDbUzBuxDV38DQ84byBmYhpq8+C/p1/LOVOtoRqWf10wQZ7a78oJXWpI6FSKGTMp+WLn/J8e9Lx//EMT5bluLecfZQ3LzHxwemYo4+JnUXwRYkSOue/dosikNpFqbfm1NeWZm8p2vnh6u9z7hiOrXuiT1Wd0eiLsyXdcz34jcHhFkEDAyQUxE73YMW9ELyapsYh5I7ahZt6Ibejib3jAeQMxE9NQ+5PLe4+yvAxI4V3Xy7CYJWMme3L7jlLxfmXNjKmxuXtnsWo2pTwzUzjgykkTlSdSjMj4xIlt8o0dwTMz/iYCpm7HCJTKduplZhAzER6YAScXxEz0Ysi8Eb2YpMYi5o3Yhpp5I7ahi7/hAecNxExMQ+0LjEfeceX/mdUm/a5In6u9WhYjaPw6mVq7mdWqcylfZmaWl5lanL39WroHtGx5rShm+VjQmhmTmTE7lt11/oRh7di/7veWrU1rVzVrZhAzER6YAScXxEz0Ysi8Eb2YpMYi5o3Yhpp5I7ahi7/hAecNxEyMQ21qZIwg+NTjh4ZtcWzeN69qu5bF2F1MjwqBgJMLYiYqARtuB/NGNOOSeKuYN2IdYuaNWIcvvsYHnDcQM/ENsWe5Sf9ePDMzzIvyzEzM3cP8KBIIOLkgZqIYvJJNzBvRjU1iLWPeiH1omTdiH8L4ORBw3ggmZr74Lz3yvvdPls7O+IHAYghAoHEE+vpEHv7lAfnKwil1G2XeqIuICyCQCgLMG6kIM05CoKEERjFvBBMzf/X9+2XOnEvk+BMaaieNQQACMSPw+msi27f/WG752O/VtZx5oy4iLoBAKggwb6QizDgJgYYSGMW8EUzMXPN/LpPOznvkAx+Y2lBDaQwCEIgXgYce6pa+vj+Rr/7+D+oazrxRFxEXQCAVBJg3UhFmnIRAQwmMYt4IJmaMdX/1b3fJtKMXyjnnIGgaGi0ag0BMCGzb1i373/4XueW/fiqwxcwbgVFxIQQSSYB5I5FhxSkIhEpglPNGcDHjCxqrbaGcNnuqHDNdqKEJNZQ0DoHWEzBrVt/aJ/Lijm5xi6MTMr71RtAwb7Q+llgAgWYRYN5oFmn6gUByCIxj3hidmDHIzNKR9rZF4jgXiOtOTg5FPKlH4Pe69sjvdLwje51O2et2lP7vlP77lttR73Y+jyMByzogmcyvZKB4a6ClZbV8ZN6IY/QbavPvdu6Trf1HS5+2GtoujUWQAPNGBIMSTZOOsgZkhtUvMzJ9MtP81+qXmZk+ebL/KLm/d2Y0jcaqcAiMY94YvZgJxwVajQGBLVu2ZF55fntWu25WtGS1Jd5/RVRWtD5OlBREpCDa/FcVRJyCzriFpbb9Wgzcw0QIQCBEAptWrdulrYH5zAchQqZpCESQwPr1649uOziQc5WVs0RntaicFslZIlktulu0yosleXGloDM6n5FM/oD0523bdiPoDiZFkABiJoJBiaNJd9j2xAOOldWZtqx2dVZEZ0VJVmudE1GdntAxIkdJQSlVEEcVBoqyffn65fvi6C82QwACoyOAmBkdL66GQJwI2LbdMdWxso6ycq6onDJ/+5UyYiWnRLpKX3TqvFJWXovOa0sKE6SYv8a2u+PkJ7ZGkwBiJppxSZRV5lsZq8/JiitZS0qTm8hgVkdJn5fFUbpgvpXxRI+lCmpSR2Hx4sXvJgoEzkAgxQQQMykOPq4nhsB6e/2pmWIxJ5aV1UrnlCs588WlaDlVlOSVSF6bLy21zmudyetMP6szEhP96DqCmIlubFJh2SZ703vcgf6cyphsTkngKKVKP4vsKWVzVEGbb3UstyBFKRxqcwq2bRdTAQgnIZAQAoiZhAQSNxJP4Gb75hlF6c1ZjsqJpbPKLAvT2lsapszfZZG8iMqLuAUlVl67bn7J2pVmmTkvCLSEAGKmJdjpNAiB9bZ9quW2m3qcrAzV50hWeYJHfuOlrU0mx5WCYwSPZUQvHnUAACAASURBVBWW2cteDNI210AAAs0lgJhpLm96g8BIBDYv2jyhf/KBXFumLes6bk4rv4ZFzMoJS0QXlKi81pJXlsoXXafQdmhCfvHNrJhgZEWPAGImejHBojoEbNu2JhQzWWWyN0bkiFUSPCbVLXJ8aW2uKijlljYksKSQGVCFa9et2AVcCECgNQQQM63hTq/pJrBpxZrTXcvKZSyd1a7KibfM26x+0CeYDItSKu9qt6C0lbcsndeqM7/YXrw73dTwPm4EEDNxixj2jkjAfNvkTDtU2oTA7LTmZXTMEjaz45pMGtpxzcvoqIISp6AybuFLtr0XtBCAQHgEEDPhsaXldBP4yrp1x/X3qpxWTk5pVaphEcmJ9upZdomXXZG8NruGuW7BaWvLs4oh3WMmad4jZpIWUfypSeBW2z7KMTuuSaa005rZca1Um2O+pXKGbSs9uBGBPthWWLppaQ9YIQCB8RFAzIyPH3enm4Bt25MnSntOXL+Gxc2JsszfLrMsrOjVsGhtdgv1dgtTliq8KwNme+PedJPD+zQQQMykIcr4WJfAV5avO67Yrs1W0iVxY/5IaLMfvrcpwdtG6JgdWsw6Ym8vfKULR594XP5zn/vcQN3GuQACEBDEDIMAAvUJ3GLfmHOKjrdbmCidE2+3MJ0TraYrJXltiu+VFEwti6Wt/IQJkv/vyznioD5ZrkgyAcRMkqOLbw0hsOHLG05W2smaXV0ss3Rt8KDQwS2md5TO0DGHhErBFdlutbv5JStXmg0KeEEAAoMEEDMMBQiUCGy0N55gSX9OO1ZO1GCGxTuTzSu+32EEiym+NzUsjqsKGUfnv7RuxcvwgwAEqhNAzDAyIDAOAptWrMlKWyZrUv/eAaGltcpmK8tZXhbH33HN7LZmztGx2gpL7aU7x9Elt0IglgQQM7EMG0aPkcCGpRum6Y7+XFsmk3VdN2e2Nx48j8UIloMmy+Jqcx6L5HXGymunWDj6zePyn/sbsv1jRM5tKSaAmElx8HE9PAK2bXd1OVZWaTW449rQ2TlZS2RKSeSogphdZEQVipYU3D6rsHz9crOHPy8IJI4AYiZxIU29Q1u2bMns2PbrnGWpnDu4S5gSndOlIvwp5gDJoRoWVxXcjM53SDG/yLbfST08AECggQQQMw2ESVMQCELAfGOnJhSzyvWXrPnbSnuCRxuho4zYEeVtK60cp9CRcQvX2HZ3kPa5BgJRJICYiWJUsCkIgQ1f/vLJSndklbieULG8DWTMAZJqjmvOYzHZFcvUsOi8qal025z8UtsmAx8ELtdAoAEEEDMNgEgTEGgUgZts+1jtmB1qMl6NTqk+x/+/2m9OXPYyOq4U1OCOa537phSu+do1fY2ygXYgEAYBxEwYVGmzUQQ22/YxA9KWy7iSdZXktKtzSplzWbz59x1/tzCtdN4SlXfbdf6Q4xRs23YbZQPtQAACYyOAmBkbN+6CQNMJ3HL92pPcdp0V1/tWMCva2+3GFzuvih5cumYEj6nPESksWbvS/JcXBFpOADHT8hCk3gDbtjsmSXvOLboloWJJVmnJeZu5aNUpSuVFdN6cy2I2drG0m3cmd+aXLmV7/tQPHgBEmgBiJtLhwTgIBCOw2V4/x3Xd0tbS3kGhRvCYg0L1Kf5Bod7W0kbkmM0IVH/huhtueCVY61wFgfETQMyMnyEtBCNw04o1s3VGvBPvldnWWHmHJpssy8mmjkV7S3l1XltWXhynMOBk8tffeP3rwVrnKghAIGoEEDNRiwj2QKCBBMw3kd5GBJk2b8c1EZ01672N2HFFjvZ2XPMyOqVMjnZUIdPemV9sL97dQDNoCgKcM8MYaCiBG5fdOLOtrZjTSuUs8b68yZXqWLxDkHd7WRatC+bUe5XR+TZpy19rL9veUCNoDAIQiAQBxEwkwoAREGg+gY0bN05Rh4olkWNqcyzv20tvi2lLidJmAwLvcDZ3uznzQFtS0F1theuuu25/862lx7gTIDMT9wg23/7NmzdP0G8fyDnKyhmxYok5yFjlxNsGXyuvjkXpvNJW3tFOwcpk8tNkIP852z7YfGvpEQIQaBUBxEyryNMvBCJM4Gb75hna6c/qjBE6OmuKYUtn6CiztXRP6aDQ0o5rylKFolMs9GZcUwzbG2G3MK2FBBAzLYQf8a7X2/acdrfdZIu9ZWHaO/XeK7x/j5hlYVqbeSavtM47Xva4mF9s22SPIx5XzINAswggZppFmn4gkBACG237RHFVVuk2b9maf1Col91RapdZuubttDZYn2NZqnCtvdxsN222neaVUgKImZQGftDtTbb9HlfacpZjCu/NRibmLBZz6r3Zkl7vLNWymAJ8t+CKlc+4bn7x2pU70k0N7yEAgSAEEDNBKHENBCAQiMCmFWtO97YyNZsQeDuuKV/szC4dFGoyOqpgHliUOUNnQBW+tG7Fy4Ea56JYE0DMxDp8gYzfuGTJFDVxWlY5gzuEeUtXjWDxiu/7zRcdRrCYc1nMXOBqNz9x5lH5a65ha/lAgLkIAhCoSgAxw8CAAARCJ3DHn9/R3nPCvqz2dlwbFDveGTpmUwI13RM6vthxjdhxCpJxC0ts+43QjaODphBAzDQFc+idaNFq04q1OdWWySrH9QrwxWxtXMqwHGOWhXmbiYjOK8vKZ0Tn+/qswvL1y/eFbhwdQAACqSSAmEll2HEaAtEhcJttT+51rKxrDgr11smXzs7xztIRafd3XPO2ljabElhScHszhWUblr0dHS+wpB4BxEw9QtH6fPP1a2cNZEzRvZvTYg7yHcqwmN/NF012RVmSd10pWBmdd3R7/robrmO792iFEWsgkAoCiJlUhBknIRBPAjcuu3F6ptP1Dgot1edYWWswo+OKHBrK6Jgtpl0paKULHUdPzl977bXmM14RIoCYiVAwBk2xbfuoyU5n1lFOTg2eem8pK2sOkVQiB1xzHosyBfgqL65byLRl8gekP2/bdjF63mARBCCQVgKImbRGHr8hEHMCG+2NJyjHycrgjmveBgReNscsd3Hf9LI4WgrmgczKWHlddAqH2hyz45obc9djaT5ipjVhs227bbJ05JyikxPLynq7hWkZPJtFJnvbG5tT771t2CWf0Zl8JtNXWGTb77TGYnqFAAQgMDoCiJnR8eJqCEAgBgRuWrNmti6aDQjM8hizy1rpLB0lao63nt+cnWO2elVScFxVcC2rsMxe9lIMXIutiYiZcEO34csbTs6ogZzrqJxlecs0PdGiRE4bzGDmjXBRg7uFtTs6f+26FbvCtYrWIQABCIRPADETPmN6gAAEIkLAfEs9oZgpCRzLiJvBHdc8wSMzzUOfUqWMTmnHNVXI9KvCohuvfz0iLsTWDMTM+ENnll12drpZx5x277o5ZQ6QLNWWmQL8t7z6Mm+3MJ13lBSUq/NL1qzIK6XYFn38+GkBAhCIKAHETEQDg1kQgEBzCdz0pZsm6cn9WXFLWZyhjE5p6Vpn+Y5ranDHtUzGLVxr228119J49oaYCRa3r371q50H97yTs5SV87KJppZlaLcw6fB2C1OmhkUXVEbltTmf5WBbYemmpT3BeuAqCEAAAskigJhJVjzxBgIQCIHAZts+xnHMkrVMVpsD/4bqczzR0+fV56jSJgTeWTqWKqgDHYXFNy9+NwRzYtkkYmZ42G5asWa2Y1nebmEiVlaJzpmlYSLqxNIOfiovls6Lq7zdwlzpyC+xl7BVeSxHP0ZDAAJhEkDMhEmXtiEAgcQTuHX5uuOdDp3VJqPjLVuzstrU6JSW/+zxlqwpVdDilsSOiL8RQap2hEqjmLnJto/VTkdWKcc7j0W7OqeU2aDCO0TyDW9JmNIFs1uYJZIfKNVubU/8Lw0OQgACEGggAcRMA2HSFAQgAIFyAuvt9adarmtOQfdqdPyMjiUq64r+zeGDQs220qqgrYHCMtt+MSkUNy/aPOHaW0vbZCdVzNxh2xP3S3vOdZxcRmWyWrk50YMHSWqlzZIwJSazogrmvxnt5nsyrtlV72BS4owfEIAABFpJADHTSvr0DQEIpJKAbduW2YhAtUlW3MEd14zg8Q4NleM9kaN1wRwUqs3W0jqT15lMYam99LW4ANuwcvWVllh/J5bYbd37bi1OmbFdWwPzldN+mVL6FhF9x+LVK5fExZ/N9vo5RSnmlKNyWumcmAyLNgdJqmO16ILydgozokUKpgC/WGzLL1+/fE9c/MNOCEAAAnElgJiJa+SwGwIQSCSBzZs3T3C6D5WWrZnaHC+jY5Ymef+e5NXkmAfm0sNzwbGk4PRZheXrl++LGpBNq9buFS0TlRJHRGW06G6t9TQlqt3JZM6I2pKqdcvXHd+ecXKSyWSVa06+V2ZrY2+bYxF5RZTkPZGpVV5ZOq8cKSxeu3JH1LhjDwQgAIE0EUDMpCna+AoBCMSawPrr1h/d3i5zdMbNimsesgezOd6GBNpRZpcrb1tpsyGBFCzXKXRl3MLnbftAKxzftGrN1aLVLaJkQln/fUrk7xevXvHZVti0cePGKZkDfTl3cLcwUV7Rfc7LsijdZ0SiNqLFNecR6bzVZuU7971RuOZrX+trhb30CQEIQAACIxNAzDBCIAABCCSAwCbbfo8M7rgm5TuuGaGjlMnaDO24Zg4LVZZVmPLa9MLn/uZzA41yf9PKNXcuWbPy0+XtbVq11uzAdZz/nhI1ULSsueVZmY0r1/5FR0Y9ssi+/olG2OIv47MslXNF55SpYfEK782yMDmqdHCqzpuDUy0teZPdapdinm22G0GfNiAAAQg0lwBiprm86Q0CEIBA0wncfP3aU8wOa9osWfMPCi3V55j/7/B3XDMHhYolBSlKYcnalWaDgsCvm+0bz9CuflaL3q609aeL1yx/2NxckZ05IiuzadW6zaL1fxeRnyxZs+IjgTsUkY22faJVzOTMUjxXqZxyJaeV5MwGC54dInnXCDet81qsvFb9hetuuOGV0fTBtRCAAAQgEG0CiJloxwfrIAABCIRJQN1i35h1vfqc0o5r+vAZOrPKDwo1552YDQnEGigste2d1YzatGrdIyJ6vogMiJZVS9as2FASNH52Rg04g1mZm1euPt9V1p2i5VQRKaqM/sPF9soHK9u91baP6pe2nOWonKtcc5hkVuvBM1m09ChlhIrKm7NZLFF519X5Q22O2S0sVVtfhzlIaBsCEIBAlAkgZqIcHWyDAAQg0CICtm13dTlWVmXU4I5rQ2fnZC2RKWbXLlMMX9qQQBW0IwXLkg9pJcuUyGQR6dYizypX/39iySUi1q2i9XeXrFlx1aZV6xaJ6JtFxDLuKZF3lGV9wBHJKcctZVeUJ6zMsrCJpcJ7s7ObzluWlVeOUyj2d+Sv23jd/hbhoVsIQAACEIgIAcRMRAKBGRCAAATiQmDDhg3TVG8xq9zBs3O8Gh1vi+m5orVRJ1OGfNFianJWa0ufqTL6G3pArVFKnSdSdo2IKyJ5cyaLaLcg2sqrjJvvG1CFFetW7IoLF+yEAAQgAIHmE0DMNJ85PUJAFsiTlw10FRdZRXVBpuh9i80LAhCAAASSQECJU+yUVzO9+k4lzuatckF3EtzCBwhElQBiJqqRwa7EEpjX8ehdTpcs3LlAT31ntpaBSYl1FccgAAEIpI6AckQm7VYy81k5NP3XVl9mQK7YKvN+lDoQOAyBJhFAzDQJNN1AwBAwQqb7JFmY/7g7FSIQgAAEIJBsAkdvV5L7vuWIVgu2yvmPJ9tbvINAawggZlrDnV5TSMAsLeufWrznyasQMikMPy5DAAIpJXDsNiUn/lI99sSh+WanP14QgECDCSBmGgyU5iBQi8B5XY/e/+qF+pI9Z2kgQQACEIBAigic902rp/2AdRHZmRQFHVebRgAx0zTUdJR2AvPaHut5+jPOZGpk0j4S8B8CEEgbgdn3q0Mzn1VLHpH5X0+b7/gLgbAJIGbCJkz7EBgksEAe01sXOfCAAAQgAIGUEZj1K0tmPWzZj8j5N6TMddyFQOgEEDOhI6YDCJQIIGYYCRCAAATSSaAkZuSGR2S+nU4CeA2B8AggZsJjS8sQGEYAMcOASBKBD2enyl1Xnib3vbBfPvsPO4Zc+9YVs+WqBTOH/t3vaNn4wOuy6r6dw665/Nxj5PP/+LJ857G98uOrz5ALT5tyxHWfnDdDbvujU2TLU28N6yNJHPElHQQQM+mIM162hgBipjXc6TWFBBAzKQx6gl2uJmaeW3q2zJnRNUyUrL70RFl68fHy8xd75JLbX/CIGMFTKWYuzk6VXfv75VN3vygPFkpnDCJmEjyAUuYaYiZlAcfdphJAzDQVN52lmQBiJs3RT57vlWKmUqCUe2wEzRc/eJx85WdvehmaamLmjGO7ZGpXRh555d0h0YOYSd64SatHiJm0Rh6/m0EAMdMMyvQBAWpmGAMJI1ApZkxWxrzO3PhMVU/N5693D3hCpZqYOX5quzz80gEvG+MvS0PMJGzQpNgdxEyKg4/roRNAzISOmA4gUCJAZoaRkCQC5WLm7if2efUzL+zuHcqqVPpq6mKMYDFip5aYMZ8Z0TOtK+MtN5s1rYOamSQNmhT7gphJcfBxPXQCiJnQEdMBBBAzjIHkEQhLzPjZGLPc7M5H9yJmkjd0UukRYiaVYcfpJhFAzDQJNN1AgMwMYyBJBMJYZuYvUTOZGyNqzE5nZqMAdjNL0shJpy+ImXTGHa+bQwAx0xzO9AIBlpkxBmJNwIiLzQtPltsf2u0V8Td6AwB/CZoPySw3O+moDulsszxRU779c6xBYnwqCSBmUhl2nG4SAcRMk0DTDQTIzDAG4kzAFy9+XUzlDmXGt/FszVwpZvzlZlM6M/LtrXsQM3EePNguiBkGAQTCI4CYCY8tLUNgGAHEDAMi7gT8M2M6MspzpZrIGOuhmZVixrTvt4WYifvIwX7EDGMAAuERQMyEx5aWIYCYYQxAAAIQgACZGcYABEIkgJgJES5NQ6CcAJkZxgMEIACBdBIgM5POuON1cwggZprDmV4gwAYAjAEIQAACKSWAmElp4HG7KQQQM03BTCcQ4NBMxgAEIACBtBJAzKQ18vjdDAKImWZQpg8ICGKGQQABCEAgrQQQM2mNPH43gwBiphmU6QMCiBnGAAQgAIHUEkDMpDb0ON4EAoiZJkCmCwgYAmwAwDiAAAQgkE4CiJl0xh2vm0MAMdMczvQCAcQMYwACEIBASgkgZlIaeNxuCgHETFMw0wkEyMwwBqJN4MdXnyEXZ6cOGfn8m4fkzI3PDP37w9mpcteVp8kLu3vlkttfaIkzxoZZ0zrkO4/tbUn/dAqBsRJAzIyVHPdBoD4BxEx9RlwBgYYQYJlZQzDSSAgEjJA549gu+dTdL8qDhW6vh8r3Vl96onzi/OnSW3TlC//08tB1IZhTs0lj00tv9cln/2FHM7ulLwiMmwBiZtwIaQACNQkgZhgcEGgSAcRMk0DTzagI+CLl6794U264dJZM6cyIycq83j0gx09tl4dfOuCJh+eWnu39/DuzJsq9z++XVfftFHPv0ouPl76iK3sOFOW7j+/z+q5879xZE+VDc6ZId68jD27vlivOnS4dGeX1Y7I/1do5YVq7XLVgptdeT58jdz+xT648b7p0tlmy8YHX5f2zJ3uZpH5He/8u7O2V9R89UWZObvcyNwieUQ0DLg6ZAGImZMA0n2oCiJlUhx/nm0kAMdNM2vQVlICf7XjfqZM9sWJEwzcvn+0JEyMoTj2mU+58dK8nFJbdu1M+ePoUMdcaEfL4tWd6wsYIic0LT5bbH9otH5077Yj3jPAwL7M8zb/HiCG/b18glbfzzqGibP7ZG959/j2mHZOZeW3/gJcl+rMtO+RDp0/1ft7y1FvyyXnTPRtZhhY0+lzXLAKImWaRpp80EkDMpDHq+NwSAoiZlmCn0zoEjFB4ctdBT6D4y8d+8YW5csfDu+XT82d44sG8/CyJnym57Re75eNnHSUbHnjdEw+mnefe6JXzT5o47D0jdnwRYoSSqbsxdS/+64ldB6WrTR1xjxE2t/3RKV6myM+++O2Ye43IMuLI1NF87Q9PkZ+/2CMXnjalZUvgGGgQGIkAYobxAYHwCCBmwmNLyxAYRgAxw4CIIgFfzFx6xjQvq5Gd0eVlOv7tubflE+fPkG/9ao9cfu4xXqbGZFPMqzKj8tPfdA9lc/zMTPl7vggxS7/KMzM+D/89/56HdhwYEkW79vcPtV0vM2PEVavqeaIYW2yKDgHETHRigSXJI4CYSV5M8SiiBBAzEQ1Mys361hWzvayMqZExNSimPsXUpZiXqUUxL39Jl785gF9n89BLPV79i3n19DryjYd2ez+bmpny98rFjF8fY2pmzOvbW/d4y8aq3ePXxOx8p19+sr20McEn582oWTNz3cXHI2ZSPp6j6j5iJqqRwa4kEEDMJCGK+BALAoiZWIQpdUb6Wy7v73VGvRVzZUbFZG+qZWb8jE4tuNXaqXdP6gKFw7EmUBIzlv2InH9DrB3BeAhEkABiJoJBwaRkEkDMJDOuSfHK7FY297gJQ+74dSojiYryLEvlzmTlu5XVY1StnXr38DkE4kQAMROnaGFr3AggZuIWMeyNLYF5bY/1PP0ZZ/LApNi6gOEQgAAEIDAGArPvV4dmPquWPCLzvz6G27kFAhAYgQBihuEBgSYROK/r0ftfvVBfsucs3aQe6QYCEIAABKJA4LxvWj3tB6yLtsr5j0fBHmyAQJIIIGaSFE18iTSBBfLkZf1Ti/c8eZU7NdKGYhwEIAABCDSMwLHblJz4S/XYE4fmz29YozQEAQgMEUDMMBgg0EQC8zoevav7JFmY/ziCponY6QoCEIBASwgcvV1J7vuWI1otICvTkhDQaQoIIGZSEGRcjBYBI2icLlm4c4Ge+s5sLdTQRCs+WAMBCEBgPASUIzJpt5KZz8qh6b+2+jIDcsVWmfej8bTJvRCAQG0CiBlGBwRaQMAsORvoKi6yiuqCTFEmt8AEuoRAUwn81ge3y44nTpLens6m9ktnEGg6ASVOsVNezfTqO5U4m7fKBaVDknhBAAKhEEDMhIKVRiEAAQhAoJzAplXrdmlrYP5S234NMhCAAAQgAIFGEUDMNIok7UAAAhCAQE0CiBkGBwQgAAEIhEEAMRMGVdqEAAQgAIFhBBAzDAgIQAACEAiDAGImDKq0CQEIQAACiBnGAAQgAAEIhE4AMRM6YjqAAAQgAAEyM4wBCEAAAhAIgwBiJgyqtAkBCEAAAmRmGAMQgAAEIBA6AcRM6IjpAAIQgAAEyMwwBiAAAQhAIAwCiJkwqNImBCAAAQiQmWEMQAACEIBA6AQQM6EjpgMIQAACECAzwxiAAAQgAIEwCCBmwqBKmxCAAAQgQGaGMQABCEAAAqETQMyEjpgOIAABCECAzAxjAAIQgAAEwiCAmAmDKm1CAAIQgACZGcYABCAAAQiETgAxEzpiOoAABCAAATIzjAEIQAACEAiDAGImDKq0CQEIQAACZGYYAxCAAAQgEDoBxEzoiOkAAhCAAATIzDAGIAABCEAgDAKImTCo0iYEIAABCJCZYQxAAAIQgEDoBBAzoSOmAwhAAAIQIDPDGIAABCAAgTAIIGbCoEqbEIAABCBAZoYxAAEIQAACoRNAzISOmA4gAAEIQIDMDGMAAhCAAATCIICYCYMqbUIAAhCAAJkZxgAEIAABCIROADETOmI6gAAEIAABMjOMAQhAAAIQCIMAYiYMqrQJAQhAAAJkZhgDEIAABCAQOgHETOiI6QACEIAABMjMMAYgAAEIQCAMAoiZMKjSJgQgAAEIkJlhDEAAAhCAQOgEEDOhI6YDCEAAAhAgM8MYgAAEIACBMAggZsKgSpsQgAAEIEBmhjEAAQhAAAKhE0DMhI6YDiAAAQhAgMwMYwACEIAABMIggJgJgyptQgACEIAAmRnGAAQgAAEIhE4AMRM6YjqAAAQgAAEyM4wBCEAAAhAIgwBiJgyqtAkBCEAAAmRmGAMQgAAEIBA6AcRM6IjpAAIQgEA6CWxetHnCtbdee8h4T2YmnWMAryEAAQiETQAxEzZh2ocABCCQQgIbVq6+0hLr78QSu617363FKTO2a2tgvnLaL1NK3yKi71i8euWSFKLBZQhAAAIQaCABxEwDYdIUBCAAAQgcJrBp1dq9omWiUuKIqIwW3a21nqZEtTuZzBnL7GXb4QUBCEAAAhAYDwHEzHjocS8EIAABCNQksGnVmqtFq1tEyYSyi/qUyN8vXr3is6CDAAQgAAEIjJcAYma8BLkfAhCAAARGEDRr3xCR4/wLlKiBomXNJSvDoIEABCAAgUYQQMw0giJtQAACEIBAVQIV2RmyMowTCEAAAhBoKAHETENx0hgEIAABCFQS2LTKz86oAYesDAMEAhCAAAQaSAAx00CYNAUBCEAAAkcS8LIzYt0qWn93yZoVV8EIAhCAAAQg0CgCiJlGkaQdCEAAAhCoSWDjl9fc4ypn5TLbZgczxgkEIAABCDSMAGKmYShpCALBCSyQJy8b6CousorqgkxRJge/kyshAAEIQCDSBJQ4xU55NdOr71TibN4qF3RH2l6Mg0DMCSBmYh5AzI8fgXkdj97ldMnCnQv01HdmaxmYFD8fsBgCEIAABKoTUI7IpN1KZj4rh6b/2urLDMgVW2Xej+AFAQiEQwAxEw5XWoVAVQJGyHSfJAvzH3engggCEIAABJJN4OjtSnLftxzRasFWOf/xZHuLdxBoDQHETGu402sKCZilZf1Ti/c8eRVCJoXhx2UIQCClBI7dpuTEX6rHnjg0f35KEeA2BEIlgJgJFS+NQ+AwgfO6Hr3/1Qv1JXvO0mCBAAQgAIEUETjvm1ZP+wHrIrIzKQo6rjaNAGKmaajpKO0E5rU91vP0Z5zJ1MikfSTgPwQgkDYCs+9Xh2Y+q5Y8IvO/njbf8RcCYRNAzIRN3lCdMgAAGXxJREFUmPYhMEhggTymty5y4AEBCEAAAikjMOtXlsx62LIfkfNvSJnruAuB0AkgZkJHTAcQKBFAzDASIAABCKSTQEnMyA2PyHw7nQTwGgLhEUDMhMeWliEwjABihgFRi8C3rpgtl597jHz+H1+W7zy2V8y/r1owU769dY989h92DN324exUuevK0+SF3b3yyx0HZOnFx0tHRlVt9vk3D8kX/ull7/pZ0zqqXuO37/dX7aJyG4LYdcntLwxr5sdXnyEXZ6fW9OW+F/YP8zHoKDG2XHrGNPnU3S/Kg4Xax3j4/fvt9vQ5Q5wr+6p3rc+/Gs8HCt1S6XtQX7gu+QQQM8mPMR62jgBipnXs6TllBBAzKQv4KNytJWYqH7zLxUz5g7P/fqUwqPV+pWm1hIF5/5PzZsjGB16XVfftHBJZQe0y925eeLJ09zrS2aaGCY+gto0kAEcSM377U7syw8RLNUEW9NpaNhs/b/ujU+SRV95F0Ixi3KfpUsRMmqKNr80mgJhpNnH6Sy0BxExqQ1/X8WpiZuFZR3v37T4wIGdufMb7udliprI/Y+do7PJF0v9++i35zIKZ8pWfvemJonJfwsrMmCzLGcd2Vc3cVPIOeu2u/f1epquazaaN46e2D8WqbtC5IFUEEDOpCjfONpkAYqbJwOkuvQQQM+mNfT3Pq4kZk3UwIuDqDxznLT0zy82iIGZGY9dzS8+W17sH5MYfvza0PM7PKIWZmanFyY9Ded93P7HvCNvK41XtWsRMvRHN55UEEDOMCQiERwAxEx5bWobAMAKIGQZELQK1xIypB1l+yQny3pMneUul/MyAqZkZzTKzajUe5UvFRrPMzF/aVc+u1ZeeKF/84HFD2ZhKH8MUM/6yry1PvVWzHscXWnc+utdbIhbkWl+UVYoZ46upX/JFJyMdAogZxgAEmkcAMdM81vSUcgKImZQPgBHcH0nMmNvM0qb9vc5QQf9oxUy9pVy1NgDod/RQvYyxo1z01LOrctlVpcCIs5hhAwB+l0dLgMzMaIlxPQSCE0DMBGfFlRAYFwHEzLjwJfrmkcSM2anL/+b/n7a9JReeNsXbzWw0mZkgYqa8mN4XHq++0z+sBqQyg1PLLv/+KZ2ZI+Jmsksm4+SLoXq2jZTNqrUBQCuWmSV6gOLcuAkgZsaNkAYgUJMAYobBAYEmEUDMNAl0DLupJ2aMSybTYYRMX9E9YtesMHYz84XKz1/sGRJO1ZajVbOr0h8/JOW7o/30N901i+mDhLDe1sxBi/rN0rCg1460AUAQm7kmvQQQM+mNPZ6HTwAxEz5jeoCARwAxw0AYKctQec5MZdah/IyTyjNNwhAzvoAqPyOmmoCoZpepRzEvfxc23+/yjEmt+pOgo6SemAm63bLpL+i1410aF9Q3rkseAcRM8mKKR9EhgJiJTiywJOEEEDMJD/A43AuSmTHNV8uWlD+M1zpnptahmeZgTSM4RhIGRpjMmdHl1c6cMK296kGV5Xb5h3nWKoY3WRCzocFtv9gtn5w3veqBnpWHhVZDW6vOx1/G5h+kWe8gzPK2612LmBnHIE/5rYiZlA8A3A+VAGImVLw0DoHDBBAzjAYIQAAC6SSAmEln3PG6OQQQM83hTC8QYJkZYwACEIBASgkgZlIaeNxuCgHETFMw0wkEqJlhDEAgKIFaS8j8+yuXkgVtl+sg0CoCRsz85e/9e6u6p18IJJqA0vLTxWtWXDSSkyrRBHAOAk0iwDKzJoGmGwhAAAIRI0BmJmIBwZzUEUDMpC7kOBwGAcRMGFRpEwIQgED0CSBmoh8jLEw2AcRMsuOLd00igJhpEmi6gQAEIBAxAoiZiAUEc1JHADGTupDjcBgEEDNhUKVNCEAAAtEngJiJfoywMNkEEDPJji/eNYkAYqZJoOkGAhCAQMQIIGYiFhDMSR0BxEzqQo7DYRBAzIRBlTYbRaDywEj/QE2/ff/AyBd298olt7/QqG5H1Y6xwRwAag7k5AWBOBFAzMQpWtiaRAKImSRGFZ+aTgAx03TkdBiQgBEyZxzbJZ+6+0V5sNDt3VX53upLT5RPnD9deouufOGfXh66LmAXDbnM2PTSW33y2X/Y0ZD2aAQCzSKAmGkWafqBQHUCiBlGBgQaQAAx0wCINNFwAr5I+fov3pQbLp0lUzozYrIyr3cPyPFT2+Xhlw544uG5pWd7P//OrIly7/P7ZdV9O8Xcu/Ti46Wv6MqeA0X57uP7PPsq3zt31kT50Jwp0t3ryIPbu+WKc6dLR0Z5/Zy58Zmq7ZwwrV2uWjDTa6+nz5G7n9gnV543XTrbLNn4wOvy/tmT5eLsVOl3tPfvwt5eWf/RE2Xm5HYvc4PgafhQocFxEEDMjAMet0KgAQQQMw2ASBMQQMwwBqJIwM92vO/UyZ5YMaLhm5fP9oSJERSnHtMpdz661xMKy+7dKR88fYqYa40IefzaMz1hY4TE5oUny+0P7ZaPzp12xHtGeJiXWZ7m32PEkN+3L5DK23nnUFE2/+wN7z7/HtOOycy8tn/AyxL92ZYd8qHTp3o/b3nqLfnkvOmejSxDi+JIS7dNiJl0xx/vW08AMdP6GGBBAgggZhIQxAS6YITCk7sOegLFXz72iy/MlTse3i2fnj/DEw/m5WdJ/EzJbb/YLR8/6yjZ8MDrnngw7Tz3Rq+cf9LEYe8ZseOLECOU7rryNK/uxX89seugdLWpI+4xwua2PzrFyxT52Re/HXOvEVlGHJk6mq/94Sny8xd75MLTprRsCVwChwYuNZAAYqaBMGkKAmMggJgZAzRugUAlAcQMYyKKBHwxc+kZ07ysRnZGl5fp+Lfn3pZPnD9DvvWrPXL5ucd4mRqTTTGvyozKT3/TPZTN8TMz5e/5IsQs/SrPzPg8/Pf8ex7acWBIFO3a3z/Udr3MjBFXrarniWJssSk6BBAz0YkFlqSTAGImnXHH6wYTQMw0GCjNNYTAt66Y7WVlTI2MqUEx9SmmLsW8TC2KeflLuvzNAfw6m4de6vHqX8yrp9eRbzy02/vZ1MyUv1cuZvw6G1MzY17f3rrHWzZW7R6/JmbnO/3yk+2ljQk+OW9GzZqZ6y4+HjHTkFFBI40mgJhpNFHag8DoCCBmRseLqyFQlQBihoERRQL+lsv7ex2vDsZ/BdmKuTKjYrI31TIzfkanlv/V2ql3TxRZYhMEahEoiRnLfkTOvwFKEIBA8wkgZprPnB4TSAAxk8CgJsgls1vZ3OMmDHnk16mMJCrKsyyVO5OV71ZWD1O1durdw+cQiBMBxEycooWtSSSAmEliVPGp6QTmtT3W8/RnnMkDk5reNR1CAAIQgEALCcy+Xx2a+axa8ojM/3oLzaBrCKSWAGImtaHH8UYSOK/r0ftfvVBfsucs3chmaQsCEIAABCJO4LxvWj3tB6yLtsr5j0fcVMyDQCIJIGYSGVacajaBBfLkZf1Ti/c8eZU7tdl90x8EIAABCLSGwLHblJz4S/XYE4fmz2+NBfQKAQggZhgDEGgQgXkdj97VfZIszH8cQdMgpDQDAQhAILIEjt6uJPd9yxGtFpCViWyYMCwFBBAzKQgyLjaPgBE0Tpcs3LlAT31nthZqaJrHnp4gAAEIhE1AOSKTdiuZ+awcmv5rqy8zIFdslXk/Crtf2ocABGoTQMwwOiDQYAJmydlAV3GRVVQXZIoyucHN0xwEIkfgtz64XXY8cZL09nRGzjYMgkBDCShxip3yaqZX36nE2bxVLigdksQLAhBoGQHETMvQ0zEEIACBZBDYtGrdLm0NzF9q268lwyO8gAAEIACBuBBAzMQlUtgJAQhAIKIEEDMRDQxmQQACEEgBAcRMCoKMixCAAATCJICYCZMubUMAAhCAwEgEEDOMDwhAAAIQGBcBxMy48HEzBCAAAQiMgwBiZhzwuBUCEIAABEQQM4wCCEAAAhBoFQHETKvI0y8EIACBhBBAzCQkkLgBAQhAIIYEEDMxDBomQwACEIgSAcRMlKKBLRCAAATSRQAxk6544y0EIACBhhNAzDQcKQ1CAAIQgEBAAoiZgKC4DAIQgAAEqhNAzDAyIAABCECgVQQQM60iT78QgAAEEkIAMZOQQOIGBCAAgRgSQMzEMGiYDAEIQCBKBBAzUYoGtkAAAhBIFwHETLrijbcQgAAEGk4AMdNwpDQIAQhAAAIBCSBmAoLiMghAAAIQqE4AMcPIgAAEIACBVhFAzLSKPP1CAAIQSAgBxExCAokbEIAABGJIADETw6BhMgQgAIEoEUDMRCka2AIBCEAgXQQQM+mKN95CAAIQaDgBxEzDkdIgBCAAAQgEJICYCQiKyyAAAQhAoDoBxAwjAwIQgAAEWkUAMdMq8vQLAQhAICEEEDMJCSRuQAACEIghAcRMDIOGyRCAAASiRAAxE6VoYAsEIACBdBFAzKQr3ngLAQhAoOEEEDMNR0qDEIAABCAQkABiJiAoLoMABCAAgeoEEDOMDAhAAAIQaBUBxEyryNMvBCAAgYQQQMwkJJC4AQEIQCCGBBAzMQwaJkMAAhCIEgHETJSigS0QgAAE0kUAMZOueOMtBCAAgYYTQMw0HCkNQgACEIBAQAKImYCguAwCEIAABKoTQMwwMiAAAQhAoFUEEDOtIk+/EIAABBJCADGTkEDiBgQgAIEYEkDMxDBomAwBCEAgSgQQM1GKBrZAAAIQSBcBxEy64o23EIAABBpOADHTcKQ0CAEIQAACAQkgZgKC4jIIQAACEKhOADHDyIAABCAAgVYRQMy0ijz9QgACEEgIAcRMQgKJGxCAAARiSAAxE8OgYTIEIACBVhPYvGjzhGtvvfaQsQMx0+po0D8EIACB9BJAzKQ39ngOAQhAYEwENqxcfaUl1t+JJXZb975bi1NmbNfWwHzltF+mlL5FRN+xePXKJWNqnJsgAAEIQAACoyCAmBkFLC6FAAQgAIESgU2r1u4VLROVEkdEZbTobq31NCWq3clkzlhmL9sOKwhAAAIQgEDYBBAzYROmfQhAAAIJJLBp1ZqrRatbRMmEMvf6lMjfL1694rMJdBmXIAABCEAgggQQMxEMCiZBAAIQiAOBTavWviEix/m2KlEDRcuaS1YmDtHDRghAAALJIICYSUYc8QICEIBA0wlUZGfIyjQ9AnQIAQhAAAKIGcYABCAAAQiMmcDh7IwacMjKjJkjN0IAAhCAwNgIIGbGxo27IAABCEDA2whgzdUi1q2i9XeXrFlxFVAgAAEIQAACzSSAmGkmbfqCAAQgkEACG7+85h5XOSuX2TY7mCUwvrgEAQhAIMoEEDNRjg62xZLA2ZfeeJmy3EXiygWiZHIsncBoCEAAAhCoQkA5IvpV0frOg8rZvP2HdjeYIACB1hJAzLSWP70njMA5l669SytZqESmJsw13IEABCAAgcMEDolInxJ9xdM/XPkjwEAAAq0jgJhpHXt6ThgBI2SUkoUaIZOwyOIOBCAAgVoElKMtZ8Ez9656HEYQgEBrCCBmWsOdXhNGwCwtE+XeQ0YmYYHFHQhAAAL1CTy27Ycr5te/jCsgAIEwCCBmwqBKm6kjcM5la+8XLZekznEchgAEIACBHm25F5GdYSBAoDUEEDOt4U6vCSNwzqVreyj2T1hQcQcCEIBAMAKHtMiSZ3644uvBLucqCECgkQQQM42kSVupJXDOf1mrU+s8jkMAAhBIOwGt7G33XX9D2jHgPwRaQQAx0wrq9Jk4AoiZxIUUhyAAAQgEJ6D1DdvuW2kHv4ErIQCBRhFAzDSKJO2kmgBiJtXhT4zzH/vw2bL885fKpAkdR/g0UHTkb7/3sDy67WVZ96WPe59ff/O/yiNPvzR0rf2XH5X//Ltz5cbb7pPvP/iM/M2Nn5AF5556RFsvvrJX/uAv7kgMNxyBgCBmGAQQaBkBxEzL0NNxkgggZpIUzfT6YsTM4j+/RLbc+4Tc9p2fVQXx3t8+1RMzx06fIlufekn+fPl3RxQzM4+ZPEy4+PcfeLcPQZPeoZY8zxEzyYspHsWGAGImNqHC0CgTQMxEOTrYFpTAaMSMESMnnXC0l63xhU+1zEylmDG2+BmgH/3H82L/9b1BzeM6CESXAGImurHBssQTQMwkPsQ42AwCiJlmUKaPsAmMRsw89Nhv5Ld/60SZPKlzaLlZUDFj/Pjn//E52fPWgWGZnbD9o30IhEYAMRMaWhqGQD0CiJl6hPgcAgEIIGYCQOKSyBMYqWZm974eT7SYl1lmZsTMY8+84tXYPPvr1zxRMhoxY+ppqmVtIg8JAyFQjQBihnEBgZYRQMy0DD0dJ4kAYiZJ0UyvL6PNzJglYkbAfOzis73lZkacVG4AUEuwIGbSO84S6TliJpFhxal4EEDMxCNOWBlxAoiZiAcI8wIRGIuYMQ2bJWNmudkTz74qF753zrDdzGqJGZaZBQoJF8WFAGImLpHCzgQSQMwkMKi41HwCiJnmM6fHxhMYq5jxl6cNDDjS3p6pK2bYAKDxsaPFFhNAzLQ4AHSfZgKImTRHH98bRgAx0zCUNNRCAmMVM8Zks9zsDz5yrrx7qH9EMcPWzC0MMF2HRwAxEx5bWoZAHQKIGYYIBBpAADHTAIg00XICI20AYIwz58p86x8eGtoAoHJbZbN07LiZU+semll5Pk3LHccACIyXAGJmvAS5HwJjJoCYGTM6boTAYQKIGUYDBCAAgRQTQMykOPi43moCiJlWR4D+E0EAMZOIMOIEBCAAgbERQMyMjRt3QaABBBAzDYBIExBAzDAGIAABCKSYAGImxcHH9VYTQMy0OgL0nwgCiJlEhBEnIAABCIyNAGJmbNy4CwINIICYaQBEmoAAYoYxAAEIQCDFBBAzKQ4+rreaAGKm1RGg/0QQQMwkIow4AQEIQGBsBBAzY+PGXRBoAAHETAMg0gQEEDOMAQhAAAIpJoCYSXHwcb3VBBAzrY4A/SeCAGImEWHECQhAAAJjI4CYGRs37oJAAwggZhoAkSYggJhhDESZwN/c+AlZcO6pQya++Mpe+YO/uGPo3+/97VO9gzB3vLpP/nz5d1viirHh2OlT5PsPPtOS/ukUAuMigJgZFz5uhsB4CCBmxkOPeyEwSAAxw1CIKgEjZGafNF2uv/lf5ZGnX/LMrHzv85/8oFx20VnS31+U9bf/+9B1zfTJ2PTam++I/df3NrNb+oJAYwggZhrDkVYgMAYCiJkxQOMWCFQSQMwwJqJIwBcp/+vfHpOr/9/flUkTOsRkZfa8dUBmHjNZnv6/Oz3x8M//43Pez2ec/h75+aPb5bbv/EzMvX/6x++T/gFH3t5/UH7wk2c9Fyvf+0+nHSfzzjlF3j3Y54mgj/zuXGlvy3j9mOxPtXZM33/wkXO99t491C8//Mmz8l8uOks62jPyt997WH77t070MkkDRcf798u73pK//NOL5OhpE+X7DzyD4IniYEu7TYiZtI8A/G8hAcRMC+HTdXIIIGaSE8skeeJnO4w4MGLlBz99Tr78lx/1hIkRFCccd5T864+3eULhr//2JzLv7JM9IWFEyP/66lWesDFCYvGfXyJb7n1CLpw/54j3zPXmZZan+fcYMeT37Quk8na6D/TKd/55q3eff49px2RmjNAyWaIb/vpemX/OKd7PP/qP5+VjF5/t2cgytCSN0AT5gphJUDBxJW4EEDNxixj2RpIAYiaSYUm9UUYovPCbNzyB4i8fu/PmT8v3fvCEfPySczzxYF5+lsTPlJhMzkUX5OTbW37piQfTzm9e2Stz57xn2HtG7PgixAglU3dj6l781//d/oZ0drQdcY8RNss/f6mXKfKzL3475l4jsow4MnU0y67+iDzx7Cty3lknt2wJXOoHEgDqE0DM1GfEFRAIiQBiJiSwNJsuAoiZdMU7Lt76YuYD8073shqnzDrGy3T8bGtePnrRWfKP9z3lLQszmRqTTTGvyozKo9teHsrm+JmZ8vd8EWKWq5VnZnxG/nv+PU89v3NIFO3e1zPUdr3MzIcuyCFm4jLw0mgnYiaNUcfniBBAzEQkEJgRbwKImXjHL6nW23/5US9zYpZumRoUU59i6lLMy9SimJe/pMvfHMCvs3nq+Vc9oWNe7x7sly33Pu79bGpmyt8rFzN+fYypmTGvf/73p7y+q93j18S8ubdHHh3cmMAsJatVM3PV5e9HzCR1oCbBL8RMEqKIDzElgJiJaeAwO1oEEDPRigfWlAj4Wy4feLdv1FsxV2ZUTPamWmbGz+jUYl6tnXr3ED8IxI6AVva2+66/IXZ2YzAEEkAAMZOAIOJC6wkgZlofAyyoTcDsVnbayTOGLvDrVEYSFeVZlsqdycp3K6vHvVo79e7hcwjEjgBiJnYhw+DkEEDMJCeWeNJCAudcurZHlExuoQl0DQEIQAACrSFwSIsseeaHK77emu7pFQLpJoCYSXf88b5BBM65bO39ouWSBjVHMxCAAAQgEB8CPdpyL3rm3lWlwjJeEIBAUwkgZpqKm86SSuDsS2+8TJR7jxKZmlQf8QsCEIAABKoSeGzbD1fMhw0EINAaAoiZ1nCn1wQSOOfStXcpJQs1giaB0cUlCEAAAtUIKEdbzgKyMowOCLSOAGKmdezpOYEEjKDRShaSoUlgcHEJAhCAwGECh0SkT4m+4ukfrvwRYCAAgdYRQMy0jj09J5SAWXKmLHeRuHIBmwIkNMi4BQEIpJSAckT0q6L1nQeVs3n7D+3ulILAbQhEhsD/D29SpI2dyHxnAAAAAElFTkSuQmCC" + } + }, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us now define the Workflow for our experiment. Here we use the methodology as provided in [quickstart](https://github.com/intel/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb), and define the workflow consisting of following steps:\n", + "-\t`start`: Start of the flow \n", + "-\t`compute_loss_and_accuracy`: Compute Train Loss and Test Accuracy on aggregated model. Performed *foreach collaborator* in Federation\n", + "-\t`gather_results_and_take_weighted_average`: Collect train loss, and test accuracy metrics for each collaborator and take weighted average to compute the *Aggregated* Train Loss and Test Accuracy. Performed on Aggregator\n", + "-\t`select_collaborators`: Randomly select *n_selected_collaborators* from the entire set of collaborators in Federation. Performed on Aggregator\n", + "-\t‘train_selected_collaborators` - Train selected collaborators on its individual datasets for *local_epoch* number of times. Performed on *n_selected_collaborators*\n", + "-\t`join`: Take weighted average of the model. Performed on Aggregator\n", + "-\t`end`: End of one round of flow. Flow can be run for *n_epochs* to obtain the desired results\n", + "\n", + "We also import the FedProxOptimizer from openfl.utilities.optimizer\n", + "\n", + "![federated-flow-diagram.png](attachment:federated-flow-diagram.png)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from openfl.experimental.interface import FLSpec, Aggregator, Collaborator\n", + "from openfl.experimental.runtime import LocalRuntime\n", + "from openfl.experimental.placement import aggregator, collaborator\n", + "from openfl.utilities.optimizers.torch import FedProxOptimizer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "class FedProxFlow(FLSpec):\n", + "\n", + " def __init__(self, model=None, optimizer=None, n_selected_collaborators=10, n_rounds=10, **kwargs):\n", + " super(FedProxFlow, self).__init__(**kwargs)\n", + " self.round_number = 1\n", + " self.n_selected_collaborators = n_selected_collaborators\n", + " self.n_rounds = n_rounds\n", + " self.loss_and_acc = {\"Train Loss\": [], \"Test Accuracy\": []}\n", + " if model is not None:\n", + " self.model = model\n", + " self.optimizer = optimizer\n", + " else:\n", + " self.model = Net()\n", + " self.optimizer = FedProxOptimizer(\n", + " self.model.parameters(), lr=learning_rate, mu=mu, weight_decay=weight_decay)\n", + "\n", + " @aggregator\n", + " def start(self):\n", + " \"\"\"\n", + " Start of the flow. Call compute_loss_and_accuracy step for each collaborator\n", + " \"\"\"\n", + " print(f'\\nStarting round number {self.round_number} .... \\n')\n", + " self.collaborators = self.runtime.collaborators\n", + " self.next(self.compute_loss_and_accuracy, foreach='collaborators')\n", + "\n", + " @collaborator\n", + " def compute_loss_and_accuracy(self):\n", + " \"\"\"\n", + " Compute training accuracy, training loss, aggregated validation accuracy,\n", + " aggregated validation loss, \n", + " \"\"\"\n", + " # Compute Train Loss and Train Acc\n", + " self.training_accuracy, self.training_loss, _, = compute_loss_and_acc(\n", + " self.model, self.train_loader)\n", + " \n", + " # Compute Test Loss and Test Acc\n", + " self.agg_validation_score, self.agg_validation_loss, test_correct = compute_loss_and_acc(\n", + " self.model, self.test_loader)\n", + "\n", + " self.train_dataset_length = len(self.train_loader.dataset)\n", + " self.test_dataset_length = len(self.test_loader.dataset)\n", + "\n", + " print(\n", + " \" | Train Round: {:<5} : Train Loss {:<.6f}, Test Acc: {:<.6f} [{}/{}]\".format(\n", + " self.input,\n", + " self.round_number,\n", + " self.training_loss,\n", + " self.agg_validation_score,\n", + " test_correct, \n", + " self.test_dataset_length\n", + " )\n", + " )\n", + "\n", + " self.next(self.gather_results_and_take_weighted_average)\n", + "\n", + " @aggregator\n", + " def gather_results_and_take_weighted_average(self, inputs):\n", + " \"\"\"\n", + " Gather results of all collaborators computed in previous \n", + " step.\n", + " Compute train and test weightes, and compute weighted average of \n", + " aggregated training loss, and aggregated test accuracy\n", + " \"\"\"\n", + " # Calculate train_weights and test_weights\n", + " train_datasize, test_datasize = [], []\n", + " for input_ in inputs:\n", + " train_datasize.append(input_.train_dataset_length)\n", + " test_datasize.append(input_.test_dataset_length)\n", + "\n", + " self.train_weights, self.test_weights = [], []\n", + " for input_ in inputs:\n", + " self.train_weights.append(input_.train_dataset_length / sum(train_datasize))\n", + " self.test_weights.append(input_.test_dataset_length / sum(test_datasize))\n", + "\n", + " aggregated_model_accuracy_list, aggregated_model_loss_list = [], []\n", + " for input_ in inputs:\n", + " aggregated_model_loss_list.append(input_.training_loss)\n", + " aggregated_model_accuracy_list.append(input_.agg_validation_score)\n", + "\n", + " # Weighted average of training loss\n", + " self.aggregated_model_training_loss = weighted_average(aggregated_model_loss_list, self.train_weights)\n", + " # Weighted average of aggregated model accuracy\n", + " self.aggregated_model_test_accuracy = weighted_average(aggregated_model_accuracy_list, self.test_weights)\n", + "\n", + " # Store experiment results\n", + " self.loss_and_acc[\"Train Loss\"].append(self.aggregated_model_training_loss)\n", + " self.loss_and_acc[\"Test Accuracy\"].append(self.aggregated_model_test_accuracy)\n", + "\n", + " print(\n", + " \" | Train Round: {:<5} : Agg Train Loss {:<.6f}, Agg Test Acc: {:<.6f}\".format(\n", + " self.round_number,\n", + " self.aggregated_model_training_loss,\n", + " self.aggregated_model_test_accuracy\n", + " )\n", + " )\n", + "\n", + " self.next(self.select_collaborators)\n", + "\n", + " @aggregator\n", + " def select_collaborators(self):\n", + " \"\"\"\n", + " Randomly select n_selected_collaborators collaborator\n", + " \"\"\"\n", + " np.random.seed(self.round_number)\n", + " self.selected_collaborator_indices = np.random.choice(range(len(self.collaborators)), \\\n", + " self.n_selected_collaborators, replace=False)\n", + " self.selected_collaborators = [self.collaborators[idx] for idx in self.selected_collaborator_indices]\n", + "\n", + " self.next(self.train_selected_collaborators, foreach=\"selected_collaborators\")\n", + "\n", + " @collaborator\n", + " def train_selected_collaborators(self):\n", + " \"\"\"\n", + " Train selected collaborators\n", + " \"\"\"\n", + " self.model.train(mode=True)\n", + "\n", + " self.train_dataset_length = len(self.train_loader.dataset)\n", + "\n", + " # Rebuild the optimizer with global model parameters\n", + " self.optimizer = FedProxOptimizer(\n", + " self.model.parameters(), lr=learning_rate, mu=mu, weight_decay=weight_decay)\n", + " # Set global model parameters as old weights to enable computation of proximal term\n", + " self.optimizer.set_old_weights([p.clone().detach() for p in self.model.parameters()])\n", + "\n", + " for epoch in range(local_epoch):\n", + " train_loss = []\n", + " correct = 0\n", + " for data, target in self.train_loader:\n", + " self.optimizer.zero_grad()\n", + " output = self.model(data)\n", + " loss = cross_entropy(output, target)\n", + " loss.backward()\n", + " self.optimizer.step()\n", + " pred = output.argmax(dim=1, keepdim=True)\n", + " tar = target.argmax(dim=1, keepdim=True)\n", + " correct += pred.eq(tar).sum().cpu().numpy()\n", + " train_loss.append(loss.item())\n", + " training_accuracy = float(correct / self.train_dataset_length)\n", + " training_loss = np.mean(train_loss)\n", + " print(\n", + " \" | Train Round: {:<5} | Local Epoch: {:<3}: FedProx Optimization Train Loss {:<.6f}, Train Acc: {:<.6f} [{}/{}]\".format(\n", + " self.input,\n", + " self.round_number,\n", + " epoch,\n", + " training_loss,\n", + " training_accuracy,\n", + " correct, \n", + " len(self.train_loader.dataset)\n", + " )\n", + " )\n", + "\n", + " self.next(self.join)\n", + " \n", + " @aggregator\n", + " def join(self, inputs):\n", + " \"\"\"\n", + " Compute train dataset, and take weighted average of model.\n", + " \"\"\"\n", + " train_datasize = sum([input_.train_dataset_length for input_ in inputs])\n", + "\n", + " train_weights, model_state_dict_list = [], [] \n", + " for input_ in inputs:\n", + " train_weights.append(input_.train_dataset_length / train_datasize)\n", + " model_state_dict_list.append(input_.model.state_dict())\n", + "\n", + " avg_model_dict = weighted_average(model_state_dict_list, train_weights)\n", + " self.model.load_state_dict(avg_model_dict)\n", + "\n", + " self.next(self.internal_loop)\n", + "\n", + " @aggregator\n", + " def internal_loop(self):\n", + " \"\"\"\n", + " Check if training is finished for `self.n_rounds`\n", + " if finished move to end step. Otherwise, go back to start\n", + " step for next round of training.\n", + " \"\"\"\n", + " if self.round_number < self.n_rounds:\n", + " self.round_number += 1\n", + " self.next(self.start)\n", + " else:\n", + " self.next(self.end)\n", + "\n", + " @aggregator\n", + " def end(self):\n", + " \"\"\"\n", + " This is the 'end' step.\n", + " \"\"\"\n", + " self.round_number += 1\n", + " print('This is end of the flow')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Federation\n", + "\n", + "In this step we define entities necessary to run the flow and create a function which returns dataset as private attributes of collaborator. As described in [quickstart](https://github.com/securefederatedai/openfl/blob/develop/openfl-tutorials/experimental/Workflow_Interface_101_MNIST.ipynb) we define entities necessary for the flow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "num_collaborators = 30\n", + "\n", + "# Setup aggregator\n", + "aggregator = Aggregator()\n", + "\n", + "# Setup collaborators with private attributes\n", + "collaborator_names = [f\"col{i}\" for i in range(num_collaborators)]\n", + "\n", + "synthetic_federated_dataset = SyntheticFederatedDataset(\n", + " batch_size=batch_size, num_classes=10, num_collaborators=len(collaborator_names), seed=RANDOM_SEED)\n", + "\n", + "def callable_to_initialize_collaborator_private_attributes(index):\n", + " return synthetic_federated_dataset.split(index)\n", + "\n", + "collaborators = []\n", + "for idx, collaborator_name in enumerate(collaborator_names):\n", + " collaborators.append(\n", + " Collaborator(\n", + " name=collaborator_name, num_cpus=0.0, num_gpus=0.0,\n", + " private_attributes_callable=callable_to_initialize_collaborator_private_attributes,\n", + " index=idx\n", + " )\n", + " )\n", + "\n", + "local_runtime = LocalRuntime(\n", + " aggregator=aggregator, collaborators=collaborators, backend=\"single_process\")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We define `loss_and_acc` dictionary to store the test results of our experiment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "loss_and_acc = {\n", + " \"FedProx\": {\n", + " \"Train Loss\": [], \"Test Accuracy\": []\n", + " },\n", + " \"FedAvg\": {\n", + " \"Train Loss\": [], \"Test Accuracy\": []\n", + " }\n", + "}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Data Distribution\n", + "\n", + "Now that our Federation is setup and actors (Aggregator & Collaborators) are initialized, let us take a moment to analyze the *Synthetic non-IID dataset*. We check how the targets for individual collaborators are distributed across each of the classes by computing and plotting the heat-map distribution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import seaborn as sns\n", + "from matplotlib.colors import LogNorm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "targets_for_collaborators = []\n", + "\n", + "for idx, collab in enumerate(collaborators):\n", + " # Train, and Test dataset is divided into 9:1 ratio\n", + " _, train_y = callable_to_initialize_collaborator_private_attributes(idx)[\"train_loader\"].dataset[:]\n", + " _, test_y = callable_to_initialize_collaborator_private_attributes(idx)[\"test_loader\"].dataset[:]\n", + " # Append train, and test into 1 tensor array\n", + " y = pt.cat((train_y, test_y))\n", + " targets = np.argmax(y.numpy(), axis = 1)\n", + " # Count number of samples for each class\n", + " frequency = np.zeros(10, dtype=np.int32)\n", + " for i, item in enumerate(targets):\n", + " frequency[item] += 1\n", + " targets_for_collaborators.append(frequency)\n", + "\n", + "result_arr = np.array(targets_for_collaborators).T.tolist()\n", + "fig, ax = plt.subplots(figsize=(20, 5))\n", + "ax = sns.heatmap(result_arr, annot=True, fmt=\"d\", annot_kws={\"fontsize\": 7}, ax=ax, norm=LogNorm(), cbar=False)\n", + "ax.set_title('Distribution of Classes in Dataset across Collaborators', fontsize=12)\n", + "ax.set_xlabel('Collaborator ID', fontsize=10)\n", + "ax.set_ylabel('Classes (0 - 9)', fontsize=10)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FedProx\n", + "\n", + "Now that we have flow and runtime defined, let's define our parameters and run the experiment with FedProxOptimizer (mu > 0)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Randomly select `n_selected_collaborators` collaborators\n", + "# Must be less than total collaborators\n", + "n_selected_collaborators = 10\n", + "n_epochs = 100\n", + "learning_rate = 0.01\n", + "weight_decay = 0.001\n", + "local_epoch = 20\n", + "\n", + "# Set `mu` to `1.0` for FedProx\n", + "mu = 1.0\n", + "\n", + "flflow = FedProxFlow(n_selected_collaborators=n_selected_collaborators, n_rounds=n_epochs, checkpoint=False)\n", + "flflow.runtime = local_runtime\n", + "\n", + "flflow.run()\n", + "loss_and_acc[\"FedProx\"][\"Train Loss\"] = flflow.loss_and_acc[\"Train Loss\"][:]\n", + "loss_and_acc[\"FedProx\"][\"Test Accuracy\"] = flflow.loss_and_acc[\"Test Accuracy\"][:]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FedAvg\n", + "\n", + "Now that we have obtained FedProx results, let's define the parameters for FedAvg and run experiment. Note that for comparison we only change the parameter mu to 0.0 (i.e. FedProxOptimizer with mu = 0.0)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mu = 0.0\n", + "\n", + "flflow = FedProxFlow(n_selected_collaborators=n_selected_collaborators, n_rounds=n_epochs, checkpoint=False)\n", + "flflow.runtime = local_runtime\n", + "\n", + "flflow.run()\n", + "loss_and_acc[\"FedAvg\"][\"Train Loss\"] = flflow.loss_and_acc[\"Train Loss\"][:]\n", + "loss_and_acc[\"FedAvg\"][\"Test Accuracy\"] = flflow.loss_and_acc[\"Test Accuracy\"][:]" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Compare Results\n", + "\n", + "Now that we have obtained results for both the optimizers available we conclude the tutorial by comparing the Aggregated Training Loss and Aggregated Test Accuracy. Reference: Appendix C.3.2, Figure 6 of [Federated Optimization in Heterogeneous Networks](https://arxiv.org/pdf/1812.06127.pdf) for Synthetic (0,0) dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 6))\n", + "fig.subplots_adjust(hspace=0.4, top=0.8)\n", + "\n", + "fedprox_loss = loss_and_acc[\"FedProx\"][\"Train Loss\"]\n", + "fedavg_loss = loss_and_acc[\"FedAvg\"][\"Train Loss\"]\n", + "ax1.plot(fedprox_loss,'gv-', label='FedProx (mu=1.0)')\n", + "ax1.plot(fedavg_loss,'rs-', label='FedAvg (mu=0.0)')\n", + "ax1.legend()\n", + "ax1.minorticks_on()\n", + "ax1.grid(which='major',linestyle='-',color='0.5')\n", + "ax1.grid(which='minor',linestyle='--',color='0.25')\n", + "ax1.set_title('Train Loss')\n", + "ax1.set_xlabel('Training Round')\n", + "ax1.set_ylabel('Training Loss')\n", + "\n", + "fedprox_accuracy = loss_and_acc[\"FedProx\"][\"Test Accuracy\"]\n", + "fedavg_accuracy = loss_and_acc[\"FedAvg\"][\"Test Accuracy\"]\n", + "ax2.plot(fedprox_accuracy,'gv-', label='FedProx (mu=1.0)')\n", + "ax2.plot(fedavg_accuracy, 'rs-', label='FedAvg (mu=0.0)')\n", + "ax2.legend()\n", + "ax2.minorticks_on()\n", + "ax2.grid(which='major',linestyle='-',color='0.5')\n", + "ax2.grid(which='minor',linestyle='--',color='0.25')\n", + "ax2.set_title('Test Accuracy')\n", + "ax2.set_xlabel('Training Round')\n", + "ax2.set_ylabel('Test Accuracy')\n", + "\n", + "fig.suptitle('Comparison of FedProx (mu > 0) and FedAvg (mu = 0)', fontsize='18')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env_fedprox_example", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.16" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "c96b31a6dd4c6365f3cc206f3a3aedb434a4eb5a8aa6c7dc735a6d54c4b635a9" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/openfl-tutorials/experimental/requirements_workflow_interface.txt b/openfl-tutorials/experimental/requirements_workflow_interface.txt index e687320c82..f091055e25 100644 --- a/openfl-tutorials/experimental/requirements_workflow_interface.txt +++ b/openfl-tutorials/experimental/requirements_workflow_interface.txt @@ -1,4 +1,5 @@ -dill==0.3.6 -metaflow==2.7.15 -ray==2.2.0 -numpy==1.21.6 \ No newline at end of file +dill==0.3.6 +metaflow==2.7.15 +ray==2.2.0 +torch +torchvision diff --git a/openfl/experimental/interface/__init__.py b/openfl/experimental/interface/__init__.py index 0942123eed..fc03bd8459 100644 --- a/openfl/experimental/interface/__init__.py +++ b/openfl/experimental/interface/__init__.py @@ -3,7 +3,7 @@ """openfl.experimental.interface package.""" -from .fl_spec import FLSpec, final_attributes +from .fl_spec import FLSpec from .participants import Aggregator, Collaborator -__all__ = ["FLSpec", "final_attributes", "Aggregator", "Collaborator"] +__all__ = ["FLSpec", "Aggregator", "Collaborator"] diff --git a/openfl/experimental/interface/fl_spec.py b/openfl/experimental/interface/fl_spec.py index 0ebce0c4a0..ee7be0c986 100644 --- a/openfl/experimental/interface/fl_spec.py +++ b/openfl/experimental/interface/fl_spec.py @@ -1,29 +1,27 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 - """openfl.experimental.interface.flspec module.""" from __future__ import annotations import inspect from copy import deepcopy -from typing import Type, List, Callable +from typing import Callable, List, Type + +from openfl.experimental.runtime import Runtime from openfl.experimental.utilities import ( MetaflowInterface, SerializationError, aggregator_to_collaborator, + checkpoint, collaborator_to_aggregator, - should_transfer, filter_attributes, - checkpoint, + generate_artifacts, + should_transfer, ) -from openfl.experimental.runtime import Runtime - -final_attributes = [] class FLSpec: - _clones = [] _initial_state = None @@ -31,7 +29,8 @@ def __init__(self, checkpoint: bool = False): """Initializes the FLSpec object. Args: - checkpoint (bool, optional): Determines whether to checkpoint or not. Defaults to False. + checkpoint (bool, optional): Determines whether to checkpoint or + not. Defaults to False. """ self._foreach_methods = [] self._checkpoint = checkpoint @@ -39,7 +38,7 @@ def __init__(self, checkpoint: bool = False): @classmethod def _create_clones(cls, instance: Type[FLSpec], names: List[str]) -> None: """Creates clones for instance for each collaborator in names. - + Args: instance (Type[FLSpec]): The instance to be cloned. names (List[str]): The list of names for the clones. @@ -55,9 +54,10 @@ def _reset_clones(cls): @classmethod def save_initial_state(cls, instance: Type[FLSpec]) -> None: """Saves the initial state of an instance before executing the flow. - + Args: - instance (Type[FLSpec]): The instance whose initial state is to be saved. + instance (Type[FLSpec]): The instance whose initial state is to be + saved. """ cls._initial_state = deepcopy(instance) @@ -65,22 +65,28 @@ def run(self) -> None: """Starts the execution of the flow.""" # Submit flow to Runtime - self._metaflow_interface = MetaflowInterface( - self.__class__, self.runtime.backend - ) - self._run_id = self._metaflow_interface.create_run() if str(self._runtime) == "LocalRuntime": - # Setup any necessary ShardDescriptors through the LocalEnvoys - # Assume that first task always runs on the aggregator - self._setup_aggregator() + self._metaflow_interface = MetaflowInterface( + self.__class__, self.runtime.backend) + self._run_id = self._metaflow_interface.create_run() + # Initialize aggregator private attributes + self.runtime.initialize_aggregator() self._foreach_methods = [] FLSpec._reset_clones() FLSpec._create_clones(self, self.runtime.collaborators) - # the start function can just be invoked locally + # Initialize collaborator private attributes + self.runtime.initialize_collaborators() if self._checkpoint: print(f"Created flow {self.__class__.__name__}") try: - self.start() + # Execute all Participant (Aggregator & Collaborator) tasks and + # retrieve the final attributes + # start step is the first task & invoked on aggregator through + # runtime.execute_task + final_attributes = self.runtime.execute_task( + self, + self.start, + ) except Exception as e: if "cannot pickle" in str(e) or "Failed to unpickle" in str(e): msg = ( @@ -90,28 +96,22 @@ def run(self) -> None: "\nLocalRuntime(...,backend='single_process')\n" "\n or for more information about the original error," "\nPlease see the official Ray documentation" - "\nhttps://docs.ray.io/en/latest/ray-core/objects/serialization.html" - ) + "\nhttps://docs.ray.io/en/releases-2.2.0/ray-core/\ + objects/serialization.html") raise SerializationError(str(e) + msg) else: raise e for name, attr in final_attributes: setattr(self, name, attr) elif str(self._runtime) == "FederatedRuntime": - raise Exception("Submission to remote runtime not available yet") + pass else: raise Exception("Runtime not implemented") - def _setup_aggregator(self): - """Sets aggregator private attributes as self attributes.""" - - for name, attr in self.runtime._aggregator.private_attributes.items(): - setattr(self, name, attr) - @property def runtime(self) -> Type[Runtime]: """Returns flow runtime. - + Returns: Type[Runtime]: The runtime of the flow. """ @@ -120,7 +120,7 @@ def runtime(self) -> Type[Runtime]: @runtime.setter def runtime(self, runtime: Type[Runtime]) -> None: """Sets flow runtime. - + Args: runtime (Type[Runtime]): The runtime to be set. @@ -139,7 +139,7 @@ def _capture_instance_snapshot(self, kwargs): kwargs: Key word arguments originally passed to the next function. If include or exclude are in the kwargs, the state of the aggregator needs to be retained. - + Returns: return_objs (list): A list of return objects. """ @@ -149,16 +149,17 @@ def _capture_instance_snapshot(self, kwargs): return_objs.append(backup) return return_objs - def _is_at_transition_point(self, f: Callable, parent_func: Callable) -> bool: + def _is_at_transition_point(self, f: Callable, + parent_func: Callable) -> bool: """Determines if the collaborator has finished its current sequence. - Has the collaborator finished its current sequence? Args: f (Callable): The next function to be executed. parent_func (Callable): The previous function executed. Returns: - bool: True if the collaborator has finished its current sequence, False otherwise. + bool: True if the collaborator has finished its current sequence, + False otherwise. """ if parent_func.__name__ in self._foreach_methods: self._foreach_methods.append(f.__name__) @@ -170,13 +171,14 @@ def _is_at_transition_point(self, f: Callable, parent_func: Callable) -> bool: return True return False - def _display_transition_logs(self, f: Callable, parent_func: Callable) -> None: - """Prints aggregator to collaborators or - collaborators to aggregator state transition logs. + def _display_transition_logs(self, f: Callable, + parent_func: Callable) -> None: + """Prints aggregator to collaborators or collaborators to aggregator + state transition logs. Args: f (Callable): The next function to be executed. - parent_func (Callable): The previous function executed. + parent_func (Callable): The previous function executed. """ if aggregator_to_collaborator(f, parent_func): print("Sending state from aggregator to collaborators") @@ -184,39 +186,112 @@ def _display_transition_logs(self, f: Callable, parent_func: Callable) -> None: elif collaborator_to_aggregator(f, parent_func): print("Sending state from collaborator to aggregator") - def next(self, f: Callable, **kwargs) -> None: + def filter_exclude_include(self, f, **kwargs): + """Filters exclude/include attributes for a given task within the flow. + + Args: + f (Callable): The task to be executed within the flow. + **kwargs (dict): Additional keyword arguments. These should + include: + - "foreach" (str): The attribute name that contains the list + of selected collaborators. + - "exclude" (list, optional): List of attribute names to + exclude. If an attribute name is present in this list and the + clone has this attribute, it will be filtered out. + - "include" (list, optional): List of attribute names to + include. If an attribute name is present in this list and the + clone has this attribute, it will be included. + """ + selected_collaborators = getattr(self, kwargs["foreach"]) + + for col in selected_collaborators: + clone = FLSpec._clones[col] + clone.input = col + if ("exclude" in kwargs and hasattr(clone, kwargs["exclude"][0]) + ) or ("include" in kwargs + and hasattr(clone, kwargs["include"][0])): + filter_attributes(clone, f, **kwargs) + artifacts_iter, _ = generate_artifacts(ctx=self) + for name, attr in artifacts_iter(): + setattr(clone, name, deepcopy(attr)) + clone._foreach_methods = self._foreach_methods + + def restore_instance_snapshot(self, ctx: FLSpec, + instance_snapshot: List[FLSpec]): + """Restores attributes from backup (in instance snapshot) to ctx. + + Args: + ctx (FLSpec): The context to restore the attributes to. + instance_snapshot (List[FLSpec]): The list of FLSpec instances + that serve as the backup. + """ + for backup in instance_snapshot: + artifacts_iter, _ = generate_artifacts(ctx=backup) + for name, attr in artifacts_iter(): + if not hasattr(ctx, name): + setattr(ctx, name, attr) + + def get_clones(self, kwargs): + """Create, and prepare clones.""" + FLSpec._reset_clones() + FLSpec._create_clones(self, self.runtime.collaborators) + selected_collaborators = self.__getattribute__(kwargs["foreach"]) + + for col in selected_collaborators: + clone = FLSpec._clones[col] + clone.input = col + artifacts_iter, _ = generate_artifacts(ctx=clone) + attributes = artifacts_iter() + for name, attr in attributes: + setattr(clone, name, deepcopy(attr)) + clone._foreach_methods = self._foreach_methods + clone._metaflow_interface = self._metaflow_interface + + def next(self, f, **kwargs): """Specifies the next task in the flow to execute. Args: f (Callable): The next task that will be executed in the flow. **kwargs: Additional keyword arguments. """ - # Get the name and reference to the calling function parent = inspect.stack()[1][3] parent_func = getattr(self, parent) - # Checkpoint current attributes (if checkpoint==True) - checkpoint(self, parent_func) + if str(self._runtime) == "LocalRuntime": + # Checkpoint current attributes (if checkpoint==True) + checkpoint(self, parent_func) # Take back-up of current state of self - agg_to_collab_ss = [] + agg_to_collab_ss = None if aggregator_to_collaborator(f, parent_func): agg_to_collab_ss = self._capture_instance_snapshot(kwargs=kwargs) + if str(self._runtime) == "FederatedRuntime": + if len(FLSpec._clones) == 0: + self.get_clones(kwargs) + # Remove included / excluded attributes from next task filter_attributes(self, f, **kwargs) - if self._is_at_transition_point(f, parent_func): - # Collaborator is done executing for now - return - - self._display_transition_logs(f, parent_func) + if str(self._runtime) == "FederatedRuntime": + if f.collaborator_step and not f.aggregator_step: + self._foreach_methods.append(f.__name__) + + if "foreach" in kwargs: + self.filter_exclude_include(f, **kwargs) + # if "foreach" in kwargs: + self.execute_task_args = ( + self, + f, + parent_func, + FLSpec._clones, + agg_to_collab_ss, + kwargs, + ) + else: + self.execute_task_args = (self, f, parent_func, kwargs) - self._runtime.execute_task( - self, - f, - parent_func, - instance_snapshot=agg_to_collab_ss, - **kwargs, - ) + elif str(self._runtime) == "LocalRuntime": + # update parameters required to execute execute_task function + self.execute_task_args = [f, parent_func, agg_to_collab_ss, kwargs] diff --git a/openfl/experimental/interface/keras/__init__.py b/openfl/experimental/interface/keras/__init__.py new file mode 100644 index 0000000000..1d7d84eb7f --- /dev/null +++ b/openfl/experimental/interface/keras/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.keras package.""" + +from .aggregation_functions import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/keras/aggregation_functions/__init__.py b/openfl/experimental/interface/keras/aggregation_functions/__init__.py new file mode 100644 index 0000000000..94708487bc --- /dev/null +++ b/openfl/experimental/interface/keras/aggregation_functions/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimenal.interface.keras.aggregation_functions package.""" + +from .weighted_average import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py b/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py new file mode 100644 index 0000000000..326e57aece --- /dev/null +++ b/openfl/experimental/interface/keras/aggregation_functions/weighted_average.py @@ -0,0 +1,13 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.keras.aggregation_functions.weighted_average package.""" + + +class WeightedAverage: + """Weighted average aggregation for keras or tensorflow.""" + + def __init__(self) -> None: + """ + WeightedAverage class for Keras or Tensorflow library. + """ + raise NotImplementedError("WeightedAverage for keras will be implemented in the future.") diff --git a/openfl/experimental/interface/participants.py b/openfl/experimental/interface/participants.py index 846bf6155b..ce6ce95cf8 100644 --- a/openfl/experimental/interface/participants.py +++ b/openfl/experimental/interface/participants.py @@ -1,84 +1,299 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -"""openfl.experimental.interface.participants module.""" - -from typing import Dict -from typing import Any - - -class Participant: - """Class for a participant. - - Attributes: - private_attributes (dict): The private attributes of the participant. - _name (str): The name of the participant. - """ - - def __init__(self, name: str = ""): - """Initializes the Participant object with an optional name. - - Args: - name (str, optional): The name of the participant. Defaults to "". - """ - self.private_attributes = {} - self._name = name - - @property - def name(self): - """Returns the name of the participant. - - Returns: - str: The name of the participant. - """ - return self._name - - @name.setter - def name(self, name: str): - """Sets the name of the participant. - - Args: - name (str): The name to be set. - """ - self._name = name - - def private_attributes(self, attrs: Dict[str, Any]) -> None: - """Set the private attributes of the participant. These attributes will - only be available within the tasks performed by the participants and - will be filtered out prior to the task's state being transfered. - - Args: - attrs (Dict[str, Any]): dictionary of ATTRIBUTE_NAME (str) -> object that will be accessible - within the participant's task. - - Example: - {'train_loader' : torch.utils.data.DataLoader(...)} - - In any task performed by this participant performed within the flow, - this attribute could be referenced with self.train_loader - """ - self.private_attributes = attrs - - -class Collaborator(Participant): - """Class for a collaborator participant, derived from the Participant class.""" - - def __init__(self, **kwargs): - """Initializes the Collaborator object with variable length arguments. - - Args: - **kwargs: Variable length argument list. - """ - super().__init__(**kwargs) - - -class Aggregator(Participant): - """Class for an aggregator participant, derived from the Participant class.""" - - def __init__(self, **kwargs): - """Initializes the Aggregator object with variable length arguments. - - Args: - **kwargs: Variable length argument list. - """ - super().__init__(**kwargs) +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.participants module.""" + +from typing import Any, Callable, Dict, Optional + + +class Participant: + """Class for a participant. + + Attributes: + private_attributes (dict): The private attributes of the participant. + _name (str): The name of the participant. + """ + + def __init__(self, name: str = ""): + """Initializes the Participant object with an optional name. + + Args: + name (str, optional): The name of the participant. Defaults to "". + """ + self.private_attributes = {} + self._name = name + + @property + def name(self): + """Returns the name of the participant. + + Returns: + str: The name of the participant. + """ + return self._name + + @name.setter + def name(self, name: str): + """Sets the name of the participant. + + Args: + name (str): The name to be set. + """ + self._name = name + + def private_attributes(self, attrs: Dict[str, Any]) -> None: + """Set the private attributes of the participant. These attributes will + only be available within the tasks performed by the participants and + will be filtered out prior to the task's state being transfered. + + Args: + attrs (Dict[str, Any]): dictionary of ATTRIBUTE_NAME (str) -> + object that will be accessible within the participant's task. + + Example: + {'train_loader' : torch.utils.data.DataLoader(...)} + + In any task performed by this participant performed within the + flow, this attribute could be referenced with self.train_loader + """ + self.private_attributes = attrs + + +class Collaborator(Participant): + """Class for a collaborator participant, derived from the Participant + class. + + Attributes: + name (str): Name of the collaborator. + private_attributes_callable (Callable): A function which returns + collaborator private attributes for each collaborator. + num_cpus (int): Specifies how many cores to use for the collaborator + step execution. + num_gpus (float): Specifies how many GPUs to use to accelerate the + collaborator step execution. + kwargs (dict): Parameters required to call private_attributes_callable + function. + """ + + def __init__(self, + name: str = "", + private_attributes_callable: Callable = None, + num_cpus: int = 0, + num_gpus: int = 0.0, + **kwargs): + """Initializes the Collaborator object. + + Create collaborator object with custom resources and a callable + function to assign private attributes. + + Args: + name (str, optional): Name of the collaborator. Defaults to "". + private_attributes_callable (Callable, optional): A function which + returns collaborator private attributes for each collaborator. + In case private_attributes are not required this can be + omitted. Defaults to None. + num_cpus (int, optional): Specifies how many cores to use for the + collaborator step execution. This will only be used if backend + is set to ray. Defaults to 0. + num_gpus (float, optional): Specifies how many GPUs to use to + accelerate the collaborator step execution. This will only be + used if backend is set to ray. Defaults to 0.0. + **kwargs (dict): Parameters required to call + private_attributes_callable function. The key of the + dictionary must match the arguments to the + private_attributes_callable. Defaults to {}. + """ + super().__init__(name=name) + self.num_cpus = num_cpus + self.num_gpus = num_gpus + self.kwargs = kwargs + + if private_attributes_callable is None: + self.private_attributes_callable = private_attributes_callable + else: + if not callable(private_attributes_callable): + raise Exception( + "private_attributes_callable parameter must be a callable" + ) + else: + self.private_attributes_callable = private_attributes_callable + + def get_name(self) -> str: + """Gets the name of the collaborator. + + Returns: + str: The name of the collaborator. + """ + return self._name + + def initialize_private_attributes(self) -> None: + """Initialize private attributes of Collaborator object by invoking the + callable specified by user.""" + if self.private_attributes_callable is not None: + self.private_attributes = self.private_attributes_callable( + **self.kwargs) + elif private_attrs: + self.private_attributes = private_attrs + + def __set_collaborator_attrs_to_clone(self, clone: Any) -> None: + """Set collaborator private attributes to FLSpec clone before + transitioning from Aggregator step to collaborator steps. + + Args: + clone (Any): The clone to set attributes to. + """ + # set collaborator private attributes as + # clone attributes + for name, attr in self.private_attributes.items(): + setattr(clone, name, attr) + + def __delete_collab_attrs_from_clone(self, clone: Any) -> None: + """Remove collaborator private attributes from FLSpec clone before + transitioning from Collaborator step to Aggregator step. + + Args: + clone (Any): The clone to remove attributes from. + """ + # Update collaborator private attributes by taking latest + # parameters from clone, then delete attributes from clone. + for attr_name in self.private_attributes: + if hasattr(clone, attr_name): + self.private_attributes.update( + {attr_name: getattr(clone, attr_name)}) + delattr(clone, attr_name) + + def execute_func(self, ctx: Any, f_name: str, callback: Callable) -> Any: + """Execute remote function f. + + Args: + ctx (Any): The context to execute the function in. + f_name (str): The name of the function to execute. + callback (Callable): The callback to execute after the function. + + Returns: + Any: The result of the function execution. + """ + self.__set_collaborator_attrs_to_clone(ctx) + + callback(ctx, f_name) + + self.__delete_collab_attrs_from_clone(ctx) + + return ctx + + +class Aggregator(Participant): + """Class for an aggregator participant, derived from the Participant + class.""" + + def __init__(self, + name: str = "", + private_attributes_callable: Callable = None, + num_cpus: int = 0, + num_gpus: int = 0.0, + **kwargs): + """Initializes the Aggregator object. + + Create aggregator object with custom resources and a callable + function to assign private attributes. + + Args: + name (str, optional): Name of the aggregator. Defaults to "". + private_attributes_callable (Callable, optional): A function which + returns aggregator private attributes. In case + private_attributes are not required this can be omitted. + Defaults to None. + num_cpus (int, optional): Specifies how many cores to use for the + aggregator step execution. This will only be used if backend + is set to ray. Defaults to 0. + num_gpus (float, optional): Specifies how many GPUs to use to + accelerate the aggregator step execution. This will only be + used if backend is set to ray. Defaults to 0.0. + **kwargs: Parameters required to call private_attributes_callable + function. The key of the dictionary must match the arguments + to the private_attributes_callable. Defaults to {}. + """ + super().__init__(name=name) + self.num_cpus = num_cpus + self.num_gpus = num_gpus + self.kwargs = kwargs + + if private_attributes_callable is None: + self.private_attributes_callable = private_attributes_callable + else: + if not callable(private_attributes_callable): + raise Exception( + "private_attributes_callable parameter must be a callable") + else: + self.private_attributes_callable = private_attributes_callable + + def get_name(self) -> str: + """Gets the name of the aggregator. + + Returns: + str: The name of the aggregator. + """ + return self.name + + def initialize_private_attributes(self) -> None: + """Initialize private attributes of Aggregator object by invoking the + callable specified by user.""" + if self.private_attributes_callable is not None: + self.private_attributes = self.private_attributes_callable( + **self.kwargs) + elif private_attrs: + self.private_attributes = private_attrs + + def __set_agg_attrs_to_clone(self, clone: Any) -> None: + """Set aggregator private attributes to FLSpec clone before transition + from Aggregator step to collaborator steps. + + Args: + clone (Any): The clone to set attributes to. + """ + # set aggregator private attributes as + # clone attributes + for name, attr in self.private_attributes.items(): + setattr(clone, name, attr) + + def __delete_agg_attrs_from_clone(self, clone: Any) -> None: + """Remove aggregator private attributes from FLSpec clone before + transition from Aggregator step to collaborator steps. + + Args: + clone (Any): The clone to remove attributes from. + """ + # Update aggregator private attributes by taking latest + # parameters from clone, then delete attributes from clone. + for attr_name in self.private_attributes: + if hasattr(clone, attr_name): + self.private_attributes.update( + {attr_name: getattr(clone, attr_name)}) + delattr(clone, attr_name) + + def execute_func(self, + ctx: Any, + f_name: str, + callback: Callable, + clones: Optional[Any] = None) -> Any: + """Executes remote function f. + + Args: + ctx (Any): The context to execute the function in. + f_name (str): The name of the function to execute. + callback (Callable): The callback to execute after the function. + clones (Optional[Any], optional): The clones to use in the + function. Defaults to None. + + Returns: + Any: The result of the function execution. + """ + self.__set_agg_attrs_to_clone(ctx) + + if clones is not None: + callback(ctx, f_name, clones) + else: + callback(ctx, f_name) + + self.__delete_agg_attrs_from_clone(ctx) + + return ctx diff --git a/openfl/experimental/interface/torch/__init__.py b/openfl/experimental/interface/torch/__init__.py new file mode 100644 index 0000000000..969f47b43a --- /dev/null +++ b/openfl/experimental/interface/torch/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.torch package.""" + +from .aggregation_functions import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/torch/aggregation_functions/__init__.py b/openfl/experimental/interface/torch/aggregation_functions/__init__.py new file mode 100644 index 0000000000..2afa83b219 --- /dev/null +++ b/openfl/experimental/interface/torch/aggregation_functions/__init__.py @@ -0,0 +1,7 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimenal.interface.torch.aggregation_functions package.""" + +from .weighted_average import WeightedAverage + +__all__ = ["WeightedAverage", ] diff --git a/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py b/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py new file mode 100644 index 0000000000..a91cadfa0d --- /dev/null +++ b/openfl/experimental/interface/torch/aggregation_functions/weighted_average.py @@ -0,0 +1,77 @@ +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.interface.torch.aggregation_functions.weighted_average package.""" + +import collections +import numpy as np +import torch as pt + + +def weighted_average(tensors, weights): + """Compute weighted average.""" + return np.average(tensors, weights=weights, axis=0) + + +class WeightedAverage: + """Weighted average aggregation.""" + + def __call__(self, objects_list, weights_list) -> np.ndarray: + """ + Compute weighted average of models, optimizers, loss, or accuracy metrics. + For taking weighted average of optimizer do the following steps: + 1. Call "_get_optimizer_state" (openfl.federated.task.runner_pt._get_optimizer_state) + pass optimizer to it, to take optimizer state dictionary. + 2. Pass optimizer state dictionaries list to here. + 3. To set the weighted average optimizer state dictionary back to optimizer, + call "_set_optimizer_state" (openfl.federated.task.runner_pt._set_optimizer_state) + and pass optimizer, device, and optimizer dictionary received in step 2. + + Args: + objects_list: List of objects for which weighted average is to be computed. + - List of Model state dictionaries , or + - List of Metrics (Loss or accuracy), or + - List of optimizer state dictionaries (following steps need to be performed) + 1. Obtain optimizer state dictionary by invoking "_get_optimizer_state" + (openfl.federated.task.runner_pt._get_optimizer_state). + 2. Create a list of optimizer state dictionary obtained in step - 1 + Invoke WeightedAverage on this list. + 3. Invoke "_set_optimizer_state" to set weighted average of optimizer + state back to optimizer (openfl.federated.task.runner_pt._set_optimizer_state). + weights_list: Weight for each element in the list. + + Returns: + dict: For model or optimizer + float: For Loss or Accuracy metrics + """ + # Check the type of first element of tensors list + if type(objects_list[0]) in (dict, collections.OrderedDict): + optimizer = False + # If __opt_state_needed found then optimizer state dictionary is passed + if "__opt_state_needed" in objects_list[0]: + optimizer = True + # Remove __opt_state_needed from all state dictionary in list, and + # check if weightedaverage of optimizer can be taken. + for tensor in objects_list: + error_msg = "Optimizer is stateless, WeightedAverage cannot be taken" + assert tensor.pop("__opt_state_needed") == "true", error_msg + + tmp_list = [] + # # Take keys in order to rebuild the state dictionary taking keys back up + for tensor in objects_list: + # Append values of each state dictionary in list + # If type(value) is Tensor then it needs to be detached + tmp_list.append(np.array([value.detach() if isinstance(value, pt.Tensor) else value + for value in tensor.values()], dtype=object)) + # Take weighted average of list of arrays + # new_params passed is weighted average of each array in tmp_list + new_params = weighted_average(tmp_list, weights_list) + new_state = {} + # Take weighted average parameters and building a dictionary + for i, k in enumerate(objects_list[0].keys()): + if optimizer: + new_state[k] = new_params[i] + else: + new_state[k] = pt.from_numpy(new_params[i].numpy()) + return new_state + else: + return weighted_average(objects_list, weights_list) diff --git a/openfl/experimental/placement/__init__.py b/openfl/experimental/placement/__init__.py index 1e2f9d42c7..05b12d50bb 100644 --- a/openfl/experimental/placement/__init__.py +++ b/openfl/experimental/placement/__init__.py @@ -3,6 +3,6 @@ """openfl.experimental.placement package.""" -from .placement import RayExecutor, make_remote, aggregator, collaborator +from .placement import aggregator, collaborator -__all__ = ["RayExecutor", "make_remote", "aggregator", "collaborator"] +__all__ = ["aggregator", "collaborator"] diff --git a/openfl/experimental/placement/placement.py b/openfl/experimental/placement/placement.py index b972f9e36c..1e9ec24397 100644 --- a/openfl/experimental/placement/placement.py +++ b/openfl/experimental/placement/placement.py @@ -2,84 +2,14 @@ # SPDX-License-Identifier: Apache-2.0 import functools -import ray -from copy import deepcopy -from openfl.experimental.utilities import ( - RedirectStdStreamContext, - GPUResourcesNotAvailableError, - get_number_of_gpus, -) from typing import Callable - -class RayExecutor: - """Class for executing tasks using Ray. - - Attributes: - remote_functions (list): The list of remote functions to be executed. - remote_contexts (list): The list of contexts for the remote functions. - """ - - def __init__(self): - """Initializes the RayExecutor object.""" - - self.remote_functions = [] - self.remote_contexts = [] - - def ray_call_put(self, ctx, func): - """Calls a remote function and puts the context into the Ray object store. - - Args: - ctx: The context to be put into the Ray object store. - func: The remote function to be called. - """ - remote_to_exec = make_remote(func, num_gpus=func.num_gpus) - ref_ctx = ray.put(ctx) - self.remote_contexts.append(ref_ctx) - self.remote_functions.append(remote_to_exec.remote(ref_ctx, func.__name__)) - del remote_to_exec - del ref_ctx - - def get_remote_clones(self): - """Retrieves the remote clones. - - Returns: - clones (list): A list of deep copied remote clones. - """ - clones = deepcopy(ray.get(self.remote_functions)) - del self.remote_functions - # Remove clones from ray object store - for ctx in self.remote_contexts: - ray.cancel(ctx) - return clones - - -def make_remote(f: Callable, num_gpus: int) -> Callable: - """Assigns a function to run in its own process using Ray. - - Args: - f (Callable): The function to be assigned. - num_gpus (int): The number of GPUs to request for a task. - - Returns: - Callable: The wrapped function. - """ - f = ray.put(f) - - @functools.wraps(f) - @ray.remote(num_gpus=num_gpus, max_calls=1) - def wrapper(*args, **kwargs): - f = getattr(args[0], args[1]) - print(f"\nRunning {f.__name__} in a new process") - f() - return args[0] - - return wrapper +from openfl.experimental.utilities import RedirectStdStreamContext def aggregator(f: Callable = None) -> Callable: - """Placement decorator that designates that the task will - run at the aggregator node. + """Placement decorator that designates that the task will run at the + aggregator node. Usage: class MyFlow(FLSpec): @@ -95,7 +25,6 @@ def agg_task(self): Returns: Callable: The decorated function. """ - print(f'Aggregator step "{f.__name__}" registered') f.is_step = True f.decorators = [] @@ -105,7 +34,6 @@ def agg_task(self): f.collaborator_step = False if f.__doc__: f.__doc__ = "" + f.__doc__ - f.num_gpus = 0 @functools.wraps(f) def wrapper(*args, **kwargs): @@ -118,13 +46,9 @@ def wrapper(*args, **kwargs): return wrapper -def collaborator( - f: Callable = None, - *, - num_gpus: float = 0 -) -> Callable: - """Placement decorator that designates that the task will - run at the collaborator node. +def collaborator(f: Callable = None) -> Callable: + """Placement decorator that designates that the task will run at the + collaborator node. Usage: class MyFlow(FLSpec): @@ -145,13 +69,12 @@ def collaborator_task(self): to the task (Default = 0). Selecting a value < 1 (0.0-1.0] will result in sharing of GPUs between tasks. 1 >= results in exclusive GPU access for the task. - + Returns: Callable: The decorated function. """ - if f is None: - return functools.partial(collaborator, num_gpus=num_gpus) + return functools.partial(collaborator) print(f'Collaborator step "{f.__name__}" registered') f.is_step = True @@ -162,12 +85,6 @@ def collaborator_task(self): f.collaborator_step = True if f.__doc__: f.__doc__ = "" + f.__doc__ - total_gpus = get_number_of_gpus() - if total_gpus < num_gpus: - GPUResourcesNotAvailableError( - f"cannot assign more than available GPUs ({total_gpus} < {num_gpus})." - ) - f.num_gpus = num_gpus @functools.wraps(f) def wrapper(*args, **kwargs): diff --git a/openfl/experimental/runtime/__init__.py b/openfl/experimental/runtime/__init__.py index 9703eb398b..488e4b53bb 100644 --- a/openfl/experimental/runtime/__init__.py +++ b/openfl/experimental/runtime/__init__.py @@ -5,6 +5,7 @@ from .runtime import Runtime from .local_runtime import LocalRuntime +from .federated_runtime import FederatedRuntime -__all__ = ["LocalRuntime", "Runtime"] +__all__ = ["FederatedRuntime", "LocalRuntime", "Runtime"] diff --git a/openfl/experimental/runtime/local_runtime.py b/openfl/experimental/runtime/local_runtime.py index 258303985f..b9e308f574 100644 --- a/openfl/experimental/runtime/local_runtime.py +++ b/openfl/experimental/runtime/local_runtime.py @@ -1,262 +1,816 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -""" openfl.experimental.runtime package LocalRuntime class.""" - -from __future__ import annotations -from copy import deepcopy -import ray -import gc -from openfl.experimental.runtime import Runtime -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from openfl.experimental.interface import Aggregator, Collaborator, FLSpec -from openfl.experimental.placement import RayExecutor -from openfl.experimental.utilities import ( - aggregator_to_collaborator, - generate_artifacts, - filter_attributes, - checkpoint, -) -from typing import List -from typing import Type -from typing import Callable - - -class LocalRuntime(Runtime): - """Class for a local runtime, derived from the Runtime class. - - Attributes: - aggregator (Type[Aggregator]): The aggregator participant. - __collaborators (dict): The collaborators, stored as a dictionary of names to participants. - backend (str): The backend that will execute the tasks. - """ - - def __init__( - self, - aggregator: Type[Aggregator] = None, - collaborators: List[Type[Collaborator]] = None, - backend: str = "single_process", - **kwargs, - ) -> None: - """Initializes the LocalRuntime object to run the flow on a single node, with an optional aggregator, - an optional list of collaborators, an optional backend, and additional keyword arguments. - - Args: - aggregator (Type[Aggregator], optional): The aggregator instance that holds private attributes. - collaborators (List[Type[Collaborator]], optional): A list of collaborators; each with their own private attributes. - backend (str, optional): The backend that will execute the tasks. Defaults to "single_process". - Available options are: - - 'single_process': (default) Executes every task within the same process - - 'ray': Executes tasks using the Ray library. Each participant runs tasks in their own isolated process. - Also supports GPU isolation using Ray's 'num_gpus' argument, which can be passed in through the collaborator - placement decorator. - - Example: - @collaborator(num_gpus=1) - def some_collaborator_task(self): - ... - - By selecting num_gpus=1, the task is guaranteed exclusive GPU access. If the system has one GPU, collaborator - tasks will run sequentially. - - Raises: - ValueError: If the provided backend value is not 'ray' or 'single_process'. - """ - super().__init__() - if backend not in ["ray", "single_process"]: - raise ValueError( - f"Invalid 'backend' value '{backend}', accepted values are " - + "'ray', or 'single_process'" - ) - if backend == "ray": - if not ray.is_initialized(): - dh = kwargs.get("dashboard_host", "127.0.0.1") - dp = kwargs.get("dashboard_port", 5252) - ray.init(dashboard_host=dh, dashboard_port=dp) - self.backend = backend - if aggregator is not None: - self.aggregator = aggregator - # List of envoys should be iterable, so that a subset can be selected at runtime - # The envoys is the superset of envoys that can be selected during the experiment - if collaborators is not None: - self.collaborators = collaborators - - @property - def aggregator(self) -> str: - """Gets the name of the aggregator. - - Returns: - str: The name of the aggregator. - """ - return self._aggregator.name - - @aggregator.setter - def aggregator(self, aggregator: Type[Aggregator]): - """Set LocalRuntime _aggregator. - - Args: - aggregator (Type[Aggregator]): The aggregator to be set. - """ - self._aggregator = aggregator - - @property - def collaborators(self) -> List[str]: - """Return names of collaborators. Don't give direct access to private attributes. - - Returns: - List[str]: The names of the collaborators. - """ - return list(self.__collaborators.keys()) - - @collaborators.setter - def collaborators(self, collaborators: List[Type[Collaborator]]): - """Set LocalRuntime collaborators. - - Args: - collaborators (List[Type[Collaborator]]): The collaborators to be set. - """ - self.__collaborators = { - collaborator.name: collaborator for collaborator in collaborators - } - - def restore_instance_snapshot( - self, - ctx: Type[FLSpec], - instance_snapshot: List[Type[FLSpec]] - ): - """Restores attributes from backup (in instance snapshot) to context (ctx). - - Args: - ctx (Type[FLSpec]): The context to restore the snapshot to. - instance_snapshot (List[Type[FLSpec]]): The snapshot of the instance to be restored. - """ - for backup in instance_snapshot: - artifacts_iter, _ = generate_artifacts(ctx=backup) - for name, attr in artifacts_iter(): - if not hasattr(ctx, name): - setattr(ctx, name, attr) - - def execute_task( - self, - flspec_obj: Type[FLSpec], - f: Callable, - parent_func: Callable, - instance_snapshot: List[Type[FLSpec]] = [], - **kwargs - ): - """Performs the execution of a task as defined by the - implementation and underlying backend (single_process, ray, etc) - on a single node. - - Args: - flspec_obj (Type[FLSpec]): Reference to the FLSpec (flow) object. Contains information - about task sequence, flow attributes, that are needed to execute a future task. - f (Callable): The next task to be executed within the flow. - parent_func (Callable): The prior task executed in the flow. - instance_snapshot (List[Type[FLSpec]], optional): A prior FLSpec state that needs to be - restored from (i.e. restoring aggregator state after collaborator execution). - **kwargs: Additional keyword arguments. - """ - from openfl.experimental.interface import ( - FLSpec, - final_attributes, - ) - - global final_attributes - - if "foreach" in kwargs: - flspec_obj._foreach_methods.append(f.__name__) - selected_collaborators = flspec_obj.__getattribute__( - kwargs["foreach"] - ) - - for col in selected_collaborators: - clone = FLSpec._clones[col] - if ( - "exclude" in kwargs and hasattr(clone, kwargs["exclude"][0]) - ) or ( - "include" in kwargs and hasattr(clone, kwargs["include"][0]) - ): - filter_attributes(clone, f, **kwargs) - artifacts_iter, _ = generate_artifacts(ctx=flspec_obj) - for name, attr in artifacts_iter(): - setattr(clone, name, deepcopy(attr)) - clone._foreach_methods = flspec_obj._foreach_methods - - for col in selected_collaborators: - clone = FLSpec._clones[col] - clone.input = col - if aggregator_to_collaborator(f, parent_func): - # remove private aggregator state - for attr in self._aggregator.private_attributes: - self._aggregator.private_attributes[attr] = getattr( - flspec_obj, attr - ) - if hasattr(clone, attr): - delattr(clone, attr) - - func = None - if self.backend == "ray": - ray_executor = RayExecutor() - for col in selected_collaborators: - clone = FLSpec._clones[col] - # Set new LocalRuntime for clone as it is required - # for calling execute_task and also new runtime - # object will not contain private attributes of - # aggregator or other collaborators - clone.runtime = LocalRuntime(backend="single_process") - for name, attr in self.__collaborators[ - clone.input - ].private_attributes.items(): - setattr(clone, name, attr) - to_exec = getattr(clone, f.__name__) - # write the clone to the object store - # ensure clone is getting latest _metaflow_interface - clone._metaflow_interface = flspec_obj._metaflow_interface - if self.backend == "ray": - ray_executor.ray_call_put(clone, to_exec) - else: - to_exec() - if self.backend == "ray": - clones = ray_executor.get_remote_clones() - FLSpec._clones.update(zip(selected_collaborators, clones)) - del ray_executor - del clones - gc.collect() - for col in selected_collaborators: - clone = FLSpec._clones[col] - func = clone.execute_next - for attr in self.__collaborators[ - clone.input - ].private_attributes: - if hasattr(clone, attr): - self.__collaborators[clone.input].private_attributes[ - attr - ] = getattr(clone, attr) - delattr(clone, attr) - # Restore the flspec_obj state if back-up is taken - self.restore_instance_snapshot(flspec_obj, instance_snapshot) - del instance_snapshot - - g = getattr(flspec_obj, func) - # remove private collaborator state - gc.collect() - g([FLSpec._clones[col] for col in selected_collaborators]) - else: - to_exec = getattr(flspec_obj, f.__name__) - to_exec() - if f.__name__ == "end": - checkpoint(flspec_obj, f) - artifacts_iter, _ = generate_artifacts(ctx=flspec_obj) - final_attributes = artifacts_iter() - - def __repr__(self): - """Returns the string representation of the LocalRuntime object. - - Returns: - str: The string representation of the LocalRuntime object. - """ - return "LocalRuntime" +# Copyright (C) 2020-2023 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +"""openfl.experimental.runtime package LocalRuntime class.""" + +from __future__ import annotations + +import gc +import importlib +import math +import os +from copy import deepcopy +from logging import getLogger +from typing import TYPE_CHECKING, Optional + +import ray + +from openfl.experimental.runtime.runtime import Runtime + +if TYPE_CHECKING: + from openfl.experimental.interface import Aggregator, Collaborator, FLSpec + +from typing import Any, Callable, Dict, List, Type + +from openfl.experimental.utilities import ( + ResourcesNotAvailableError, + aggregator_to_collaborator, + check_resource_allocation, + checkpoint, + filter_attributes, + generate_artifacts, + get_number_of_gpus, +) + + +class RayExecutor: + """Class for executing tasks using the Ray framework.""" + + def __init__(self): + """Initializes the RayExecutor object.""" + self.__remote_contexts = [] + + def ray_call_put( + self, + participant: Any, + ctx: Any, + f_name: str, + callback: Callable, + clones: Optional[Any] = None, + ) -> None: + """Execute f_name from inside participant (Aggregator or Collaborator) + class with the context of clone (ctx). + + Args: + participant (Any): The participant (Aggregator or Collaborator) to + execute the function in. + ctx (Any): The context to execute the function in. + f_name (str): The name of the function to execute. + callback (Callable): The callback to execute after the function. + clones (Optional[Any], optional): The clones to use in the + function. Defaults to None. + """ + if clones is not None: + self.__remote_contexts.append( + participant.execute_func.remote(ctx, f_name, callback, clones)) + else: + self.__remote_contexts.append( + participant.execute_func.remote(ctx, f_name, callback)) + + def ray_call_get(self) -> List[Any]: + """Get remote clones and delete ray references of clone (ctx) and, + reclaim memory. + + Returns: + List[Any]: The list of remote clones. + """ + clones = ray.get(self.__remote_contexts) + del self.__remote_contexts + self.__remote_contexts = [] + + return clones + + +def ray_group_assign(collaborators, num_actors=1): + """Assigns collaborators to resource groups which share a CUDA context. + + Args: + collaborators (list): The list of collaborators. + num_actors (int, optional): Number of actors to distribute + collaborators to. Defaults to 1. + + Returns: + list: A list of GroupMember instances. + """ + + class GroupMember: + """A utility class that manages the collaborator and its group. + + This class maintains compatibility with runtime execution by assigning + attributes for each function in the Collaborator interface in + conjunction with RemoteHelper. + """ + + def __init__(self, collaborator_actor, collaborator): + """Initializes a new instance of the GroupMember class. + + Args: + collaborator_actor: The collaborator actor. + collaborator: The collaborator. + """ + from openfl.experimental.interface import Collaborator + + all_methods = [ + method for method in dir(Collaborator) + if callable(getattr(Collaborator, method)) + ] + external_methods = [ + method for method in all_methods if (method[0] != "_") + ] + self.collaborator_actor = collaborator_actor + self.collaborator = collaborator + for method in external_methods: + setattr( + self, + method, + RemoteHelper(self.collaborator_actor, self.collaborator, + method), + ) + + class RemoteHelper: + """A utility class to maintain compatibility with RayExecutor. + + This class returns a lambda function that uses + collaborator_actor.execute_from_col to run a given function from the + given collaborator. + """ + + # once ray_grouped replaces the current ray runtime this class can be + # replaced with a funtion that returns the lambda funtion, using a + # funtion is necesary because this is used in setting multiple + # funtions in a loop and lambda takes the reference to self.f_name and + # not the value so we need to change scope to avoid self.f_name from + # changing as the loop progresses + def __init__(self, collaborator_actor, collaborator, f_name) -> None: + """Initializes a new instance of the RemoteHelper class. + + Args: + collaborator_actor: The collaborator actor. + collaborator: The collaborator. + f_name (str): The name of the function. + """ + self.f_name = f_name + self.collaborator_actor = collaborator_actor + self.collaborator = collaborator + self.f = lambda *args, **kwargs: self.collaborator_actor.execute_from_col.remote( + self.collaborator, self.f_name, *args, **kwargs) + + def remote(self, *args, **kwargs): + """Executes the function with the given arguments and keyword + arguments. + + Args: + *args: The arguments to pass to the function. + **kwargs: The keyword arguments to pass to the function. + + Returns: + The result of the function execution. + """ + return self.f(*args, *kwargs) + + collaborator_ray_refs = [] + collaborators_per_group = math.ceil(len(collaborators) / num_actors) + times_called = 0 + # logic to sort collaborators by gpus, if collaborators have the same + # number of gpu then they are sorted by cpu + cpu_magnitude = len(str(abs(max([i.num_cpus for i in collaborators])))) + min_gpu = min([i.num_gpus for i in collaborators]) + min_gpu = max(min_gpu, 0.0001) + collaborators_sorted_by_gpucpu = sorted( + collaborators, + key=lambda x: x.num_gpus / min_gpu * 10**cpu_magnitude + x.num_cpus, + ) + initializations = [] + + for collaborator in collaborators_sorted_by_gpucpu: + # initialize actor group + if times_called % collaborators_per_group == 0: + max_num_cpus = max([ + i.num_cpus for i in + collaborators_sorted_by_gpucpu[times_called:times_called + + collaborators_per_group] + ]) + max_num_gpus = max([ + i.num_gpus for i in + collaborators_sorted_by_gpucpu[times_called:times_called + + collaborators_per_group] + ]) + print(f"creating actor with {max_num_cpus}, {max_num_gpus}") + collaborator_actor = ( + ray.remote(RayGroup).options( + num_cpus=max_num_cpus, + num_gpus=max_num_gpus) # max_concurrency=max_concurrency) + .remote()) + # add collaborator to actor group + initializations.append(collaborator_actor.append.remote(collaborator)) + + times_called += 1 + + # append GroupMember to output list + collaborator_ray_refs.append( + GroupMember(collaborator_actor, collaborator.get_name())) + # Wait for all collaborators to be created on actors + ray.get(initializations) + + return collaborator_ray_refs + + +class RayGroup: + """A Ray actor that manages a group of collaborators. + + This class allows for the execution of functions from a specified + collaborator using the execute_from_col method. The collaborators are + stored in a dictionary where the key is the collaborator's name. + """ + + def __init__(self): + """Initializes a new instance of the RayGroup class.""" + self.collaborators = {} + + def append( + self, + collaborator: Collaborator, + ): + """Appends a new collaborator to the group. + + Args: + name (str): The name of the collaborator. + private_attributes_callable (Callable): A callable that sets the + private attributes of the collaborator. + **kwargs: Additional keyword arguments. + """ + from openfl.experimental.interface import Collaborator + + if collaborator.private_attributes_callable is not None: + self.collaborators[collaborator.name] = Collaborator( + name=collaborator.name, + private_attributes_callable=collaborator. + private_attributes_callable, + **collaborator.kwargs, + ) + elif collaborator.private_attributes is not None: + self.collaborators[collaborator.name] = Collaborator( + name=collaborator.name, + **collaborator.kwargs, + ) + self.collaborators[ + collaborator.name].initialize_private_attributes( + collaborator.private_attributes) + + def execute_from_col(self, name, internal_f_name, *args, **kwargs): + """Executes a function from a specified collaborator. + + Args: + name (str): The name of the collaborator. + internal_f_name (str): The name of the function to execute. + *args: Additional arguments to pass to the function. + **kwargs: Additional keyword arguments to pass to the function. + + Returns: + The result of the function execution. + """ + f = getattr(self.collaborators[name], internal_f_name) + return f(*args, **kwargs) + + def get_collaborator(self, name): + """Retrieves a collaborator from the group by name. + + Args: + name (str): The name of the collaborator. + + Returns: + The collaborator instance. + """ + return self.collaborators[name] + + +class LocalRuntime(Runtime): + """Class for a local runtime, derived from the Runtime class. + + Attributes: + aggregator (Type[Aggregator]): The aggregator participant. + __collaborators (dict): The collaborators, stored as a dictionary of + names to participants. + backend (str): The backend that will execute the tasks. + """ + + def __init__( + self, + aggregator: Dict = None, + collaborators: Dict = None, + backend: str = "single_process", + **kwargs, + ) -> None: + """Initializes the LocalRuntime object to run the flow on a single + node, with an optional aggregator, an optional list of collaborators, + an optional backend, and additional keyword arguments. + + Args: + aggregator (Type[Aggregator], optional): The aggregator instance + that holds private attributes. + collaborators (List[Type[Collaborator]], optional): A list of + collaborators; each with their own private attributes. + backend (str, optional): The backend that will execute the tasks. + Defaults to "single_process". + Available options are: + - 'single_process': (default) Executes every task within the + same process. + - 'ray': Executes tasks using the Ray library. We use ray + actors called RayGroups to runs tasks in their own isolated + process. Each participant is distributed into a ray group. + The RayGroups run concurrently while participants in the + group run serially. + The default is 1 RayGroup and can be changed by using the + num_actors=1 kwarg. By using more RayGroups more concurency + is allowed with the trade off being that each RayGroup has + extra memory overhead in the form of extra CUDA CONTEXTS. + + Also the ray runtime supports GPU isolation using Ray's + 'num_gpus' argument, which can be passed in through the + collaborator placement decorator. + + Raises: + ValueError: If the provided backend value is not 'ray' or + 'single_process'. + + Example: + @collaborator(num_gpus=1) + def some_collaborator_task(self): + # Task implementation + ... + + By selecting num_gpus=1, the task is guaranteed exclusive GPU + access. If the system has one GPU, collaborator tasks will run + sequentially. + """ + super().__init__() + if backend not in ["ray", "single_process"]: + raise ValueError( + f"Invalid 'backend' value '{backend}', accepted values are " + + "'ray', or 'single_process'") + if backend == "ray": + if not ray.is_initialized(): + dh = kwargs.get("dashboard_host", "127.0.0.1") + dp = kwargs.get("dashboard_port", 5252) + ray.init(dashboard_host=dh, dashboard_port=dp) + + self.num_actors = kwargs.get("num_actors", 1) + self.backend = backend + self.logger = getLogger(__name__) + if aggregator is not None: + self.aggregator = self.__get_aggregator_object(aggregator) + + if collaborators is not None: + self.collaborators = self.__get_collaborator_object(collaborators) + + def __get_aggregator_object(self, aggregator: Type[Aggregator]) -> Any: + """Get aggregator object based on localruntime backend. + + If the backend is 'single_process', it returns the aggregator directly. + If the backend is 'ray', it creates a Ray actor for the aggregator + with the specified resources. + + Args: + aggregator (Type[Aggregator]): The aggregator class to instantiate. + + Returns: + Any: The aggregator object or a reference to the Ray actor + representing the aggregator. + + Raises: + ResourcesNotAvailableError: If the requested resources exceed the + available resources. + """ + + if aggregator.private_attributes and aggregator.private_attributes_callable: + self.logger.warning( + 'Warning: Aggregator private attributes ' + + 'will be initialized via callable and ' + + 'attributes via aggregator.private_attributes ' + + 'will be ignored') + + if self.backend == "single_process": + return aggregator + + total_available_cpus = os.cpu_count() + total_available_gpus = get_number_of_gpus() + + agg_cpus = aggregator.num_cpus + agg_gpus = aggregator.num_gpus + + if agg_gpus > 0: + check_resource_allocation( + total_available_gpus, + {aggregator.get_name(): agg_gpus}, + ) + + if total_available_gpus < agg_gpus: + raise ResourcesNotAvailableError( + f"cannot assign more than available GPUs \ + ({agg_gpus} < {total_available_gpus}).") + if total_available_cpus < agg_cpus: + raise ResourcesNotAvailableError( + f"cannot assign more than available CPUs \ + ({agg_cpus} < {total_available_cpus}).") + + interface_module = importlib.import_module( + "openfl.experimental.interface") + aggregator_class = getattr(interface_module, "Aggregator") + + aggregator_actor = ray.remote(aggregator_class).options( + num_cpus=agg_cpus, num_gpus=agg_gpus) + + if aggregator.private_attributes_callable is not None: + aggregator_actor_ref = aggregator_actor.remote( + name=aggregator.get_name(), + private_attributes_callable=aggregator. + private_attributes_callable, + **aggregator.kwargs, + ) + elif aggregator.private_attributes is not None: + aggregator_actor_ref = aggregator_actor.remote( + name=aggregator.get_name(), + **aggregator.kwargs, + ) + aggregator_actor_ref.initialize_private_attributes.remote( + aggregator.private_attributes) + + return aggregator_actor_ref + + def __get_collaborator_object(self, collaborators: List) -> Any: + """Get collaborator object based on localruntime backend. + + If the backend is 'single_process', it returns the list of + collaborators directly. + If the backend is 'ray', it assigns collaborators to Ray actors using + the ray_group_assign function. + + Args: + collaborators (List[Type[Collaborator]]): The list of collaborator + classes to instantiate. + + Returns: + Any: The list of collaborator objects or a list of references to + the Ray actors representing the collaborators. + + Raises: + ResourcesNotAvailableError: If the requested resources exceed the + available resources. + """ + for collab in collaborators: + if collab.private_attributes and collab.private_attributes_callable: + self.logger.warning( + f'Warning: Collaborator {collab.name} private attributes ' + + 'will be initialized via callable and ' + + 'attributes via collaborator.private_attributes ' + + 'will be ignored' + ) + + if self.backend == "single_process": + return collaborators + + total_available_cpus = os.cpu_count() + total_required_cpus = sum( + [collaborator.num_cpus for collaborator in collaborators]) + if total_available_cpus < total_required_cpus: + raise ResourcesNotAvailableError( + f"cannot assign more than available CPUs \ + ({total_required_cpus} < {total_available_cpus}).") + + if self.backend == "ray": + collaborator_ray_refs = ray_group_assign( + collaborators, num_actors=self.num_actors) + return collaborator_ray_refs + + @property + def aggregator(self) -> str: + """Gets the name of the aggregator. + + Returns: + str: The name of the aggregator. + """ + return self._aggregator.name + + @aggregator.setter + def aggregator(self, aggregator: Type[Aggregator]): + """Set LocalRuntime _aggregator. + + Args: + aggregator (Type[Aggregator]): The aggregator to be set. + """ + self._aggregator = aggregator + + @property + def collaborators(self) -> List[str]: + """Return names of collaborators. Don't give direct access to private + attributes. + + Returns: + List[str]: The names of the collaborators. + """ + return list(self.__collaborators.keys()) + + @collaborators.setter + def collaborators(self, collaborators: List[Type[Collaborator]]): + """Set LocalRuntime collaborators. + + Args: + collaborators (List[Type[Collaborator]]): The collaborators to be + set. + """ + if self.backend == "single_process": + + def get_collab_name(collab): + return collab.get_name() + + else: + + def get_collab_name(collab): + return ray.get(collab.get_name.remote()) + + self.__collaborators = { + get_collab_name(collaborator): collaborator + for collaborator in collaborators + } + + def get_collaborator_kwargs(self, collaborator_name: str): + """Returns kwargs of collaborator. + + Args: + collaborator_name: Collaborator name for which kwargs is to be + returned + + Returns: + kwargs: Collaborator private_attributes_callable function name, and + arguments required to call it. + """ + collab = self.__collaborators[collaborator_name] + kwargs = {} + if hasattr(collab, "private_attributes_callable"): + if collab.private_attributes_callable is not None: + kwargs.update(collab.kwargs) + kwargs["private_attributes_callable"] = ( + collab.private_attributes_callable.__name__) + + return kwargs + + def initialize_aggregator(self): + """Initialize aggregator private attributes.""" + if self.backend == "single_process": + self._aggregator.initialize_private_attributes() + else: + ray.get(self._aggregator.initialize_private_attributes.remote()) + + def initialize_collaborators(self): + """Initialize collaborator private attributes.""" + if self.backend == "single_process": + + def init_private_attrs(collab): + return collab.initialize_private_attributes() + + else: + + def init_private_attrs(collab): + return ray.get(collab.initialize_private_attributes.remote()) + + for collaborator in self.__collaborators.values(): + init_private_attrs(collaborator) + + def restore_instance_snapshot(self, ctx: Type[FLSpec], + instance_snapshot: List[Type[FLSpec]]): + """Restores attributes from backup (in instance snapshot) to context + (ctx). + + Args: + ctx (Type[FLSpec]): The context to restore the snapshot to. + instance_snapshot (List[Type[FLSpec]]): The snapshot of the + instance to be restored. + """ + for backup in instance_snapshot: + artifacts_iter, _ = generate_artifacts(ctx=backup) + for name, attr in artifacts_iter(): + if not hasattr(ctx, name): + setattr(ctx, name, attr) + + def execute_agg_steps(self, + ctx: Any, + f_name: str, + clones: Optional[Any] = None): + """Execute aggregator steps until at transition point. + + Args: + ctx (Any): The context in which the function is executed. + f_name (str): The name of the function to be executed. + clones (Optional[Any], optional): Clones if any. Defaults to None. + """ + if clones is not None: + f = getattr(ctx, f_name) + f(clones) + else: + not_at_transition_point = True + while not_at_transition_point: + f = getattr(ctx, f_name) + f() + + f, parent_func = ctx.execute_task_args[:2] + if (aggregator_to_collaborator(f, parent_func) + or f.__name__ == "end"): + not_at_transition_point = False + + f_name = f.__name__ + + def execute_collab_steps(self, ctx: Any, f_name: str): + """Execute collaborator steps until at transition point. + + Args: + ctx (Any): The context in which the function is executed. + f_name (str): The name of the function to be executed. + """ + not_at_transition_point = True + while not_at_transition_point: + f = getattr(ctx, f_name) + f() + + f, parent_func = ctx.execute_task_args[:2] + if ctx._is_at_transition_point(f, parent_func): + not_at_transition_point = False + + f_name = f.__name__ + + def execute_task(self, flspec_obj: Type[FLSpec], f: Callable, **kwargs): + """Defines which function to be executed based on name and kwargs. + + Updates the arguments and executes until end is not reached. + + Args: + flspec_obj: Reference to the FLSpec (flow) object. Contains + information about task sequence, flow attributes. + f: The next task to be executed within the flow. + + Returns: + artifacts_iter: Iterator with updated sequence of values + """ + parent_func = None + instance_snapshot = None + self.join_step = False + + while f.__name__ != "end": + if "foreach" in kwargs: + flspec_obj = self.execute_collab_task(flspec_obj, f, + parent_func, + instance_snapshot, + **kwargs) + else: + flspec_obj = self.execute_agg_task(flspec_obj, f) + f, parent_func, instance_snapshot, kwargs = ( + flspec_obj.execute_task_args) + else: + flspec_obj = self.execute_agg_task(flspec_obj, f) + f = flspec_obj.execute_task_args[0] + + checkpoint(flspec_obj, f) + artifacts_iter, _ = generate_artifacts(ctx=flspec_obj) + return artifacts_iter() + + def execute_agg_task(self, flspec_obj, f): + """Performs execution of aggregator task. + + Args: + flspec_obj: Reference to the FLSpec (flow) object. + f: The task to be executed within the flow. + + Returns: + flspec_obj: updated FLSpec (flow) object. + """ + from openfl.experimental.interface import FLSpec + + aggregator = self._aggregator + clones = None + + if self.join_step: + clones = [ + FLSpec._clones[col] for col in self.selected_collaborators + ] + self.join_step = False + + if self.backend == "ray": + ray_executor = RayExecutor() + ray_executor.ray_call_put( + aggregator, + flspec_obj, + f.__name__, + self.execute_agg_steps, + clones, + ) + flspec_obj = ray_executor.ray_call_get()[0] + del ray_executor + else: + aggregator.execute_func(flspec_obj, f.__name__, + self.execute_agg_steps, clones) + + gc.collect() + return flspec_obj + + def execute_collab_task(self, flspec_obj, f, parent_func, + instance_snapshot, **kwargs): + """Performs execution of collaborator task. + + Performs + 1. Filter include/exclude + 2. Set runtime, collab private attributes , metaflow_interface + 3. Execution of all collaborator for each task + 4. Remove collaborator private attributes + 5. Execute the next function after transition + + Args: + flspec_obj: Reference to the FLSpec (flow) object. + f: The task to be executed within the flow. + parent_func: The prior task executed in the flow. + instance_snapshot: A prior FLSpec state that needs to be restored. + + Returns: + flspec_obj: updated FLSpec (flow) object + """ + + from openfl.experimental.interface import FLSpec + + flspec_obj._foreach_methods.append(f.__name__) + selected_collaborators = getattr(flspec_obj, kwargs["foreach"]) + self.selected_collaborators = selected_collaborators + + # filter exclude/include attributes for clone + self.filter_exclude_include(flspec_obj, f, selected_collaborators, + **kwargs) + + if self.backend == "ray": + ray_executor = RayExecutor() + # set runtime,collab private attributes and metaflowinterface + for col in selected_collaborators: + clone = FLSpec._clones[col] + # Set new LocalRuntime for clone as it is required + # new runtime object will not contain private attributes of + # aggregator or other collaborators + clone.runtime = LocalRuntime(backend="single_process") + + # write the clone to the object store + # ensure clone is getting latest _metaflow_interface + clone._metaflow_interface = flspec_obj._metaflow_interface + + for collab_name in selected_collaborators: + clone = FLSpec._clones[collab_name] + collaborator = self.__collaborators[collab_name] + + if self.backend == "ray": + ray_executor.ray_call_put(collaborator, clone, f.__name__, + self.execute_collab_steps) + else: + collaborator.execute_func(clone, f.__name__, + self.execute_collab_steps) + + if self.backend == "ray": + clones = ray_executor.ray_call_get() + FLSpec._clones.update(zip(selected_collaborators, clones)) + clone = clones[0] + del clones + + flspec_obj.execute_task_args = clone.execute_task_args + + # Restore the flspec_obj state if back-up is taken + self.restore_instance_snapshot(flspec_obj, instance_snapshot) + del instance_snapshot + + gc.collect() + # Setting the join_step to indicate to aggregator to collect clones + self.join_step = True + return flspec_obj + + def filter_exclude_include(self, flspec_obj, f, selected_collaborators, + **kwargs): + """This function filters exclude/include attributes. + + Args: + flspec_obj: Reference to the FLSpec (flow) object. + f: The task to be executed within the flow. + selected_collaborators: all collaborators. + """ + + from openfl.experimental.interface import FLSpec + + for col in selected_collaborators: + clone = FLSpec._clones[col] + clone.input = col + if ("exclude" in kwargs and hasattr(clone, kwargs["exclude"][0]) + ) or ("include" in kwargs + and hasattr(clone, kwargs["include"][0])): + filter_attributes(clone, f, **kwargs) + artifacts_iter, _ = generate_artifacts(ctx=flspec_obj) + for name, attr in artifacts_iter(): + setattr(clone, name, deepcopy(attr)) + clone._foreach_methods = flspec_obj._foreach_methods + + def __repr__(self): + """Returns the string representation of the LocalRuntime object. + + Returns: + str: The string representation of the LocalRuntime object. + """ + return "LocalRuntime" diff --git a/openfl/experimental/utilities/__init__.py b/openfl/experimental/utilities/__init__.py index 1f16c5ba2c..2272d1459a 100644 --- a/openfl/experimental/utilities/__init__.py +++ b/openfl/experimental/utilities/__init__.py @@ -9,7 +9,11 @@ aggregator_to_collaborator, collaborator_to_aggregator, ) -from .exceptions import SerializationError, GPUResourcesNotAvailableError +from .exceptions import ( + SerializationError, + ResourcesNotAvailableError, + ResourcesAllocationError, +) from .stream_redirect import ( RedirectStdStreamBuffer, RedirectStdStream, @@ -21,6 +25,7 @@ generate_artifacts, filter_attributes, checkpoint, + check_resource_allocation, ) @@ -30,7 +35,8 @@ "aggregator_to_collaborator", "collaborator_to_aggregator", "SerializationError", - "GPUResourcesNotAvailableError", + "ResourcesNotAvailableError", + "ResourcesAllocationError", "RedirectStdStreamBuffer", "RedirectStdStream", "RedirectStdStreamContext", @@ -39,4 +45,5 @@ "generate_artifacts", "filter_attributes", "checkpoint", + "check_resource_allocation", ] diff --git a/openfl/experimental/utilities/exceptions.py b/openfl/experimental/utilities/exceptions.py index 38ce13966b..e925792eed 100644 --- a/openfl/experimental/utilities/exceptions.py +++ b/openfl/experimental/utilities/exceptions.py @@ -1,12 +1,26 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 + class SerializationError(Exception): """Raised when there is an error in serialization process.""" def __init__(self, *args: object) -> None: + """Initializes the SerializationError with the provided arguments. + + Args: + *args (object): Variable length argument list. """ - Initializes the SerializationError with the provided arguments. + super().__init__(*args) + pass + + +class ResourcesNotAvailableError(Exception): + """Exception raised when the required resources are not available.""" + + def __init__(self, *args: object) -> None: + """Initializes the ResourcesNotAvailableError with the provided + arguments. Args: *args (object): Variable length argument list. @@ -15,11 +29,13 @@ def __init__(self, *args: object) -> None: pass -class GPUResourcesNotAvailableError(Exception): - """Raised when the required GPU resources are not available.""" +class ResourcesAllocationError(Exception): + """Exception raised when there is an error in the resources allocation + process.""" def __init__(self, *args: object) -> None: - """Initializes the GPUResourcesNotAvailableError with the provided arguments. + """Initializes the ResourcesAllocationError with the provided + arguments. Args: *args (object): Variable length argument list. diff --git a/openfl/experimental/utilities/metaflow_utils.py b/openfl/experimental/utilities/metaflow_utils.py index f78b967e18..a49305878b 100644 --- a/openfl/experimental/utilities/metaflow_utils.py +++ b/openfl/experimental/utilities/metaflow_utils.py @@ -485,8 +485,6 @@ def __init__(self, flow: Type[FLSpec], backend: str = "ray"): """ self.backend = backend self.flow_name = flow.__name__ - self._graph = FlowGraph(flow) - self._steps = [getattr(flow, node.name) for node in self._graph] if backend == "ray": self.counter = Counter.remote() else: diff --git a/openfl/experimental/utilities/resources.py b/openfl/experimental/utilities/resources.py index 1b7a19249b..bf775aef49 100644 --- a/openfl/experimental/utilities/resources.py +++ b/openfl/experimental/utilities/resources.py @@ -1,19 +1,35 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 - """openfl.experimental.utilities.resources module.""" -from torch.cuda import device_count +from logging import getLogger +from subprocess import PIPE, run +logger = getLogger(__name__) -def get_number_of_gpus(): - """Gets the number of GPUs available on the current device. - Returns: - device_count (int): The number of GPUs available on the current device. +def get_number_of_gpus() -> int: + """Returns number of NVIDIA GPUs attached to the machine. - .. note:: - This function currently depends on PyTorch. Future work includes removing this dependency. + This function executes the `nvidia-smi --list-gpus` command to get the + list of GPUs. + If the command fails (e.g., NVIDIA drivers are not installed), it logs a + warning and returns 0. + + Returns: + int: The number of NVIDIA GPUs attached to the machine. """ - # TODO remove pytorch dependency - return device_count() + # Execute the nvidia-smi command. + command = "nvidia-smi --list-gpus" + try: + op = run(command.strip().split(), + shell=False, + stdout=PIPE, + stderr=PIPE) + stdout = op.stdout.decode().strip() + return len(stdout.split("\n")) + except FileNotFoundError: + logger.warning( + f'No GPUs found! If this is a mistake please try running "{command}" ' + + "manually.") + return 0 diff --git a/openfl/experimental/utilities/runtime_utils.py b/openfl/experimental/utilities/runtime_utils.py index 426945ab13..fcfa0c94c3 100644 --- a/openfl/experimental/utilities/runtime_utils.py +++ b/openfl/experimental/utilities/runtime_utils.py @@ -3,8 +3,11 @@ """openfl.experimental.utilities package.""" +import itertools import inspect +import numpy as np from types import MethodType +from openfl.experimental.utilities import ResourcesAllocationError def parse_attrs(ctx, exclude=[], reserved_words=["next", "runtime", "input"]): @@ -112,7 +115,9 @@ def checkpoint(ctx, parent_func, chkpnt_reserved_words=["next", "runtime"]): if ctx._checkpoint: # all objects will be serialized using Metaflow interface print(f"Saving data artifacts for {parent_func.__name__}") - artifacts_iter, _ = generate_artifacts(ctx=ctx, reserved_words=chkpnt_reserved_words) + artifacts_iter, _ = generate_artifacts( + ctx=ctx, reserved_words=chkpnt_reserved_words + ) task_id = ctx._metaflow_interface.create_task(parent_func.__name__) ctx._metaflow_interface.save_artifacts( artifacts_iter(), @@ -122,3 +127,65 @@ def checkpoint(ctx, parent_func, chkpnt_reserved_words=["next", "runtime"]): buffer_err=step_stderr, ) print(f"Saved data artifacts for {parent_func.__name__}") + + +def old_check_resource_allocation(num_gpus, each_participant_gpu_usage): + remaining_gpu_memory = {} + # TODO for each GPU the funtion tries see if all participant usages fit into a GPU, it it + # doesn't it removes that + # participant from the participant list, and adds it to the remaining_gpu_memory dict. So any + # sum of GPU requirements above 1 + # triggers this. + # But at this point the funtion will raise an error because remaining_gpu_memory is never + # cleared. + # The participant list should remove the participant if it fits in the gpu and save the + # partipant if it doesn't and continue + # to the next GPU to see if it fits in that one, only if we run out of GPUs should this + # funtion raise an error. + for gpu in np.ones(num_gpus, dtype=int): + for i, (participant_name, participant_gpu_usage) in enumerate( + each_participant_gpu_usage.items() + ): + if gpu == 0: + break + if gpu < participant_gpu_usage: + remaining_gpu_memory.update({participant_name: gpu}) + each_participant_gpu_usage = dict( + itertools.islice(each_participant_gpu_usage.items(), i) + ) + else: + gpu -= participant_gpu_usage + if len(remaining_gpu_memory) > 0: + raise ResourcesAllocationError( + f"Failed to allocate Participant {list(remaining_gpu_memory.keys())} " + + "to specified GPU. Please try allocating lesser GPU resources to participants" + ) + + +def check_resource_allocation(num_gpus, each_participant_gpu_usage): + # copy participant dict + need_assigned = each_participant_gpu_usage.copy() + # cycle through all available GPU availability + for gpu in np.ones(num_gpus, dtype=int): + # buffer to cycle though since need_assigned will change sizes as we assign participants + current_dict = need_assigned.copy() + for i, (participant_name, participant_gpu_usage) in enumerate( + current_dict.items() + ): + if gpu == 0: + break + if gpu < participant_gpu_usage: + # participant doesn't fitm break to next GPU + break + else: + # if participant fits remove from need_assigned + need_assigned.pop(participant_name) + gpu -= participant_gpu_usage + + # raise error if after going though all gpus there are still participants that needed to be + # assigned + if len(need_assigned) > 0: + raise ResourcesAllocationError( + f"Failed to allocate Participant {list(need_assigned.keys())} " + + "to specified GPU. Please try allocating lesser GPU resources to participants" + ) diff --git a/openfl/experimental/utilities/stream_redirect.py b/openfl/experimental/utilities/stream_redirect.py index 815c1e24fe..458b5d8edb 100644 --- a/openfl/experimental/utilities/stream_redirect.py +++ b/openfl/experimental/utilities/stream_redirect.py @@ -68,6 +68,9 @@ def write(self, message): self.__stdDestination.write(message) self.__stdBuffer.write(message) + def flush(self): + pass + class RedirectStdStreamContext: """Context Manager that enables redirection of stdout and stderr. diff --git a/openfl/experimental/utilities/ui.py b/openfl/experimental/utilities/ui.py index 068c8f376a..d91b395ad5 100644 --- a/openfl/experimental/utilities/ui.py +++ b/openfl/experimental/utilities/ui.py @@ -1,7 +1,7 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -from openfl.experimental.utilities.metaflow_utils import DefaultCard +from openfl.experimental.utilities.metaflow_utils import DefaultCard, FlowGraph from pathlib import Path import os import webbrowser @@ -37,7 +37,10 @@ def __init__( self.show_html = show_html self.run_id = run_id self.flow_name = flow_obj.__class__.__name__ - self.graph_dict, _ = flow_obj._metaflow_interface._graph.output_steps() + self._graph = FlowGraph(flow_obj.__class__) + self._steps = [getattr(flow_obj, node.name) for node in self._graph] + + self.graph_dict, _ = self._graph.output_steps() self.show_ui() def get_pathspec(self): diff --git a/openfl/experimental/utilities/utils.py b/openfl/experimental/utilities/utils.py deleted file mode 100644 index 00581f0003..0000000000 --- a/openfl/experimental/utilities/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (C) 2020-2023 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -from torch.cuda import device_count - - -def get_number_of_gpus() -> int: - """Gets the number of GPUs available. - - Returns: - int: The number of GPUs available. - """ - return device_count() diff --git a/setup.py b/setup.py index 845d960813..1b3b14ac74 100644 --- a/setup.py +++ b/setup.py @@ -103,6 +103,10 @@ def run(self): 'openfl.databases.utilities', 'openfl.experimental', 'openfl.experimental.interface', + 'openfl.experimental.interface.keras', + 'openfl.experimental.interface.keras.aggregation_functions', + 'openfl.experimental.interface.torch', + 'openfl.experimental.interface.torch.aggregation_functions', 'openfl.experimental.placement', 'openfl.experimental.runtime', 'openfl.experimental.utilities', diff --git a/tests/github/experimental/requirements_experimental_localruntime_tests.txt b/tests/github/experimental/requirements_experimental_localruntime_tests.txt new file mode 100644 index 0000000000..3e2ac0622f --- /dev/null +++ b/tests/github/experimental/requirements_experimental_localruntime_tests.txt @@ -0,0 +1,5 @@ +dill==0.3.6 +metaflow==2.7.15 +ray==2.2.0 +torch +torchvision \ No newline at end of file diff --git a/tests/github/experimental/testflow_datastore_cli.py b/tests/github/experimental/testflow_datastore_cli.py index 40c728f0c0..9b40f765cf 100644 --- a/tests/github/experimental/testflow_datastore_cli.py +++ b/tests/github/experimental/testflow_datastore_cli.py @@ -284,31 +284,42 @@ def display_validate_errors(validate_flow_error): if __name__ == "__main__": # Setup participants - aggregator = Aggregator() - aggregator.private_attributes = {} + aggregator_ = Aggregator() # Setup collaborators with private attributes collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] - collaborators = [Collaborator(name=name) for name in collaborator_names] - - for idx, collab in enumerate(collaborators): - local_train = deepcopy(mnist_train) - local_test = deepcopy(mnist_test) - local_train.data = mnist_train.data[idx:: len(collaborators)] - local_train.targets = mnist_train.targets[idx:: len(collaborators)] - local_test.data = mnist_test.data[idx:: len(collaborators)] - local_test.targets = mnist_test.targets[idx:: len(collaborators)] - collab.private_attributes = { + + def callable_to_initialize_collaborator_private_attributes( + n_collaborators, index, train_dataset, test_dataset, batch_size + ): + local_train = deepcopy(train_dataset) + local_test = deepcopy(test_dataset) + local_train.data = mnist_train.data[index::n_collaborators] + local_train.targets = mnist_train.targets[index::n_collaborators] + local_test.data = mnist_test.data[index::n_collaborators] + local_test.targets = mnist_test.targets[index::n_collaborators] + return { "train_loader": torch.utils.data.DataLoader( - local_train, batch_size=batch_size_train, shuffle=True + local_train, batch_size=batch_size, shuffle=True ), "test_loader": torch.utils.data.DataLoader( - local_test, batch_size=batch_size_train, shuffle=True + local_test, batch_size=batch_size, shuffle=True ), } + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, num_cpus=0, num_gpus=0.0, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + n_collaborators=len(collaborator_names), index=idx, train_dataset=mnist_train, + test_dataset=mnist_test, batch_size=32 + ) + ) + local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend="ray" + aggregator=aggregator_, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") num_rounds = 5 diff --git a/tests/github/experimental/testflow_exclude.py b/tests/github/experimental/testflow_exclude.py index c2c180a112..1b5b14f1ff 100644 --- a/tests/github/experimental/testflow_exclude.py +++ b/tests/github/experimental/testflow_exclude.py @@ -85,9 +85,7 @@ def test_exclude_agg_to_collab(self): + f"{bcolors.ENDC}" ) else: - TestFlowExclude.exclude_error_list.append( - "test_exclude_agg_to_collab" - ) + TestFlowExclude.exclude_error_list.append("test_exclude_agg_to_collab") print( f"{bcolors.FAIL} ... Exclude test failed in test_exclude_agg_to_collab " + f"{bcolors.ENDC}" @@ -119,9 +117,7 @@ def test_exclude_collab_to_collab(self): + f"{bcolors.ENDC}" ) else: - TestFlowExclude.exclude_error_list.append( - "test_exclude_collab_to_collab" - ) + TestFlowExclude.exclude_error_list.append("test_exclude_collab_to_collab") print( f"{bcolors.FAIL} ... Exclude test failed in test_exclude_collab_to_collab " + f"{bcolors.ENDC}" @@ -154,23 +150,15 @@ def join(self, inputs): ) if validation: - print( - f"{bcolors.OKGREEN} ... Exclude test passed in join {bcolors.ENDC}" - ) + print(f"{bcolors.OKGREEN} ... Exclude test passed in join {bcolors.ENDC}") else: TestFlowExclude.exclude_error_list.append("join") - print( - f"{bcolors.FAIL} ... Exclude test failed in join {bcolors.ENDC}" - ) + print(f"{bcolors.FAIL} ... Exclude test failed in join {bcolors.ENDC}") - print( - f"\n{bcolors.UNDERLINE}Exclude attribute test summary: {bcolors.ENDC}\n" - ) + print(f"\n{bcolors.UNDERLINE}Exclude attribute test summary: {bcolors.ENDC}\n") if TestFlowExclude.exclude_error_list: - validated_exclude_variables = ", ".join( - TestFlowExclude.exclude_error_list - ) + validated_exclude_variables = ", ".join(TestFlowExclude.exclude_error_list) print( f"{bcolors.FAIL}...Test case failed for {validated_exclude_variables} " + f"{bcolors.ENDC}" @@ -199,20 +187,19 @@ def end(self): if __name__ == "__main__": # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = ["Portland", "Chandler", "Bangalore", "Delhi"] - collaborators = [Collaborator(name=name) for name in collaborator_names] + collaborators = [] + for collaborator_name in collaborator_names: + collaborators.append(Collaborator(name=collaborator_name)) - local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators - ) + local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") diff --git a/tests/github/experimental/testflow_include.py b/tests/github/experimental/testflow_include.py index ea4455df85..8403b557c3 100644 --- a/tests/github/experimental/testflow_include.py +++ b/tests/github/experimental/testflow_include.py @@ -87,9 +87,7 @@ def test_include_agg_to_collab(self): + f"{bcolors.ENDC}" ) else: - TestFlowInclude.include_error_list.append( - "test_include_agg_to_collab" - ) + TestFlowInclude.include_error_list.append("test_include_agg_to_collab") print( f"{bcolors.FAIL} ... Include test failed in test_include_agg_to_collab " + f"{bcolors.ENDC}" @@ -119,9 +117,7 @@ def test_include_collab_to_collab(self): + f"{bcolors.ENDC}" ) else: - TestFlowInclude.include_error_list.append( - "test_include_collab_to_collab" - ) + TestFlowInclude.include_error_list.append("test_include_collab_to_collab") print( f"{bcolors.FAIL} ... Include test failed in test_include_collab_to_collab " + f"{bcolors.ENDC}" @@ -154,23 +150,15 @@ def join(self, inputs): ) if validation: - print( - f"{bcolors.OKGREEN} ... Include test passed in join {bcolors.ENDC}" - ) + print(f"{bcolors.OKGREEN} ... Include test passed in join {bcolors.ENDC}") else: TestFlowInclude.include_error_list.append("join") - print( - f"{bcolors.FAIL} ... Include test failed in join {bcolors.ENDC}" - ) + print(f"{bcolors.FAIL} ... Include test failed in join {bcolors.ENDC}") - print( - f"\n{bcolors.UNDERLINE}Include attribute test summary: {bcolors.ENDC}\n" - ) + print(f"\n{bcolors.UNDERLINE}Include attribute test summary: {bcolors.ENDC}\n") if TestFlowInclude.include_error_list: - validated_include_variables = ",".join( - TestFlowInclude.include_error_list - ) + validated_include_variables = ",".join(TestFlowInclude.include_error_list) print( f"{bcolors.FAIL} ...Test case failed for {validated_include_variables} " + f"{bcolors.ENDC}" @@ -199,20 +187,22 @@ def end(self): if __name__ == "__main__": # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = ["Portland", "Chandler", "Bangalore", "Delhi"] - collaborators = [Collaborator(name=name) for name in collaborator_names] + collaborators = [] + for collaborator_name in collaborator_names: + collaborators.append(Collaborator(name=collaborator_name)) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") diff --git a/tests/github/experimental/testflow_include_exclude.py b/tests/github/experimental/testflow_include_exclude.py index d88bc2a2bc..7561605d10 100644 --- a/tests/github/experimental/testflow_include_exclude.py +++ b/tests/github/experimental/testflow_include_exclude.py @@ -39,9 +39,7 @@ def start(self): self.exclude_agg_to_agg = 10 self.include_agg_to_agg = 100 - self.next( - self.test_include_exclude_agg_to_agg, exclude=["exclude_agg_to_agg"] - ) + self.next(self.test_include_exclude_agg_to_agg, exclude=["exclude_agg_to_agg"]) @aggregator def test_include_exclude_agg_to_agg(self): @@ -202,24 +200,30 @@ def end(self): if __name__ == "__main__": # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = ["Portland", "Chandler", "Bangalore", "Delhi"] - collaborators = [Collaborator(name=name) for name in collaborator_names] + collaborators = [] + for collaborator_name in collaborator_names: + collaborators.append( + Collaborator( + name=collaborator_name, + ) + ) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") - flflow = TestFlowIncludeExclude(checkpoint=False) + flflow = TestFlowIncludeExclude(checkpoint=True) flflow.runtime = local_runtime for i in range(5): print(f"Starting round {i}...") diff --git a/tests/github/experimental/testflow_internalloop.py b/tests/github/experimental/testflow_internalloop.py index 6c39bd6d7f..140bae5214 100644 --- a/tests/github/experimental/testflow_internalloop.py +++ b/tests/github/experimental/testflow_internalloop.py @@ -50,9 +50,7 @@ def agg_model_mean(self): """ self.agg_mean_value = np.mean(self.model) - print( - f": {self.input} Mean of Agg model: {self.agg_mean_value} " - ) + print(f": {self.input} Mean of Agg model: {self.agg_mean_value} ") self.next(self.collab_model_update) @collaborator @@ -78,9 +76,7 @@ def join(self, inputs): """ Joining inputs from collaborators """ - self.agg_mean = sum(input.local_mean_value for input in inputs) / len( - inputs - ) + self.agg_mean = sum(input.local_mean_value for input in inputs) / len(inputs) print(f"Aggregated mean : {self.agg_mean}") self.next(self.internal_loop) @@ -212,12 +208,10 @@ def display_validate_errors(validate_flow_error): if __name__ == "__main__": - # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = [ "Portland", "Seattle", @@ -228,16 +222,16 @@ def display_validate_errors(validate_flow_error): "London", "New York", ] - collaborators = [Collaborator(name=name) for name in collaborator_names] + collaborators = [] + for collaborator_name in collaborator_names: + collaborators.append(Collaborator(name=collaborator_name)) - local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators - ) + local_runtime = LocalRuntime(aggregator=aggregator, collaborators=collaborators) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") diff --git a/tests/github/experimental/testflow_privateattributes.py b/tests/github/experimental/testflow_privateattributes.py index 30320e15e3..11173a8432 100644 --- a/tests/github/experimental/testflow_privateattributes.py +++ b/tests/github/experimental/testflow_privateattributes.py @@ -1,10 +1,11 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +import sys +import numpy as np from openfl.experimental.interface import FLSpec, Aggregator, Collaborator from openfl.experimental.runtime import LocalRuntime from openfl.experimental.placement import aggregator, collaborator -import numpy as np class bcolors: # NOQA: N801 @@ -72,9 +73,7 @@ def collaborator_step_a(self): self.exclude_collab_to_collab = 2 self.include_collab_to_collab = 22 - self.next( - self.collaborator_step_b, exclude=["exclude_collab_to_collab"] - ) + self.next(self.collaborator_step_b, exclude=["exclude_collab_to_collab"]) @collaborator def collaborator_step_b(self): @@ -105,17 +104,17 @@ def join(self, inputs): + f" not accessible {bcolors.ENDC}" ) - for input in enumerate(inputs): + for idx, collab in enumerate(inputs): if ( - hasattr(input, "train_loader") is True - or hasattr(input, "test_loader") is True + hasattr(collab, "train_loader") is True + or hasattr(collab, "test_loader") is True ): # Error - we are able to access collaborator attributes TestFlowPrivateAttributes.error_list.append( "join_collaborator_attributes_found" ) print( - f"{bcolors.FAIL} ... Attribute test failed in Join - COllaborator: {collab}" + f"{bcolors.FAIL} ... Attribute test failed in Join - Collaborator: {collab}" + f" private attributes accessible {bcolors.ENDC}" ) @@ -175,10 +174,7 @@ def validate_collab_private_attr(self, private_attr, step_name): def validate_agg_private_attrs(self, private_attr_1, private_attr_2, step_name): # Collaborator should only be able to access its own attributes - if ( - hasattr(self, private_attr_1) is False - or hasattr(self, private_attr_2) is False - ): + if hasattr(self, private_attr_1) is False or hasattr(self, private_attr_2) is False: TestFlowPrivateAttributes.error_list.append( step_name + "collab_attributes_not_found" ) @@ -199,13 +195,17 @@ def validate_agg_private_attrs(self, private_attr_1, private_attr_2, step_name): if __name__ == "__main__": - # Setup Aggregator with private attributes + # Setup Aggregator with private attributes via callable function aggregator = Aggregator() - aggregator.private_attributes = { - "test_loader": np.random.rand(10, 28, 28) # Random data - } - # Setup collaborators with private attributes + def callable_to_initialize_aggregator_private_attributes(): + return {"test_loader": np.random.rand(10, 28, 28)} # Random data + + aggregator = Aggregator( + name="agg", + private_attributes_callable=callable_to_initialize_aggregator_private_attributes, + ) + # Setup collaborators with private attributes via callable function collaborator_names = [ "Portland", "Seattle", @@ -218,19 +218,33 @@ def validate_agg_private_attrs(self, private_attr_1, private_attr_2, step_name): "Beijing", "Tokyo", ] - collaborators = [Collaborator(name=name) for name in collaborator_names] - for idx, collab in enumerate(collaborators): - collab.private_attributes = { + + def callable_to_initialize_collaborator_private_attributes(index): + return { "train_loader": np.random.rand(idx * 50, 28, 28), "test_loader": np.random.rand(idx * 10, 28, 28), } + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + index=idx, + ) + ) + + backend = "single_process" + if len(sys.argv) > 1 and sys.argv[1] == "ray": + backend = "ray" + local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, collaborators=collaborators, backend=backend ) print(f"Local runtime collaborators = {local_runtime.collaborators}") - flflow = TestFlowPrivateAttributes(checkpoint=False) + flflow = TestFlowPrivateAttributes(checkpoint=True) flflow.runtime = local_runtime for i in range(5): print(f"Starting round {i}...") diff --git a/tests/github/experimental/testflow_reference.py b/tests/github/experimental/testflow_reference.py index 8dab8a37bb..ed93d1c7ca 100644 --- a/tests/github/experimental/testflow_reference.py +++ b/tests/github/experimental/testflow_reference.py @@ -65,7 +65,6 @@ def start(self): @aggregator def test_create_agg_attr(self): - """ Create different types of objects. """ @@ -93,7 +92,6 @@ def test_create_agg_attr(self): @collaborator def test_create_collab_attr(self): - """ Modify the attirbutes of aggregator to validate the references. Create different types of objects. @@ -109,16 +107,14 @@ def test_create_collab_attr(self): self.collab_attr_str_one = "Test string data in collab " + self.input self.collab_attr_list_one = [1, 2, 5, 6, 7, 8] self.collab_attr_dict_one = {key: key for key in range(5)} - self.collab_attr_file_one = io.StringIO( - "Test file data in collaborator" - ) + self.collab_attr_file_one = io.StringIO("Test file data in collaborator") self.collab_attr_math_one = math.sqrt(self.index) self.collab_attr_complex_num_one = complex(self.index, self.index) self.collab_attr_log_one = logging.getLogger( "Test logger data in collaborator " + self.input ) - # append self attributes of collaborators + # append attributes of collaborator TestFlowReference.step_one_collab_attrs.append(self) if len(TestFlowReference.step_one_collab_attrs) >= 2: @@ -180,7 +176,7 @@ def join(self, inputs): all_shared_attr = "" print(f"\n{bcolors.UNDERLINE}Reference test summary: {bcolors.ENDC}\n") - for key, val in TestFlowReference.all_ref_error_dict.items(): + for val in TestFlowReference.all_ref_error_dict.values(): all_shared_attr = all_shared_attr + ",".join(val) if all_shared_attr: print( @@ -230,33 +226,36 @@ def filter_attrs(attr_list): return valid_attrs -def find_matched_references(collab_attr_list, all_collborators): +def find_matched_references(collab_attr_list, all_collaborators): """ Iterate attributes of collborator and capture the duplicate reference + return: dict: { + 'Portland': ['failed attributes'], 'Seattle': [], + } """ matched_ref_dict = {} - previous_collaborator = "" - # Initialize dictionary with collborator as key and value as empty list - # to hold duplicated attr list - for collborator_name in all_collborators: - matched_ref_dict[collborator_name.input] = [] - - # Iterate the attributes and get duplicate attribute id - for attr in collab_attr_list: - di = {attr: []} - for collab in all_collborators: - attr_id = id(getattr(collab, attr)) - collaborator_name = collab.input - if attr_id not in di.get(attr): - di.get(attr).append(attr_id) - else: - # append the dict with collabartor as key and attrs as value having same reference - matched_ref_dict.get(collaborator_name).append(attr) - print( - f"{bcolors.FAIL} ... Reference test failed - {collaborator_name} sharing same " - + f"{attr} reference with {previous_collaborator} {bcolors.ENDC}" - ) - previous_collaborator = collaborator_name + for i in range(len(all_collaborators)): + matched_ref_dict[all_collaborators[i].input] = [] + + # For each attribute in the collaborator attribute list, check if any of the collaborator + # attributes are shared with another collaborator + for attr_name in collab_attr_list: + for i, curr_collab in enumerate(all_collaborators): + # Compare the current collaborator with the collaborator(s) that come(s) after it. + for next_collab in all_collaborators[i + 1:]: + # Check if both collaborators have the current attribute + if hasattr(curr_collab, attr_name) and hasattr(next_collab, attr_name): + # Check if both collaborators are sharing same reference + if getattr(curr_collab, attr_name) is getattr( + next_collab, attr_name + ): + matched_ref_dict[curr_collab.input].append(attr_name) + print( + f"{bcolors.FAIL} ... Reference test failed - {curr_collab.input} \ + sharing same " + + f"{attr_name} reference with {next_collab.input} {bcolors.ENDC}" + ) + return matched_ref_dict @@ -274,9 +273,9 @@ def validate_collab_references(matched_ref_dict): if collborators_sharing_ref: for collab in collborators_sharing_ref: if collab not in TestFlowReference.all_ref_error_dict: - TestFlowReference.all_ref_error_dict[ + TestFlowReference.all_ref_error_dict[collab] = matched_ref_dict.get( collab - ] = matched_ref_dict.get(collab) + ) if not reference_flag: print( @@ -291,9 +290,7 @@ def validate_agg_attr_ref(agg_attrs, agg_obj): """ attr_flag = False for attr in agg_attrs: - if TestFlowReference.agg_attr_dict.get(attr) == id( - getattr(agg_obj, attr) - ): + if TestFlowReference.agg_attr_dict.get(attr) == id(getattr(agg_obj, attr)): attr_flag = True if not attr_flag: print( @@ -308,7 +305,6 @@ def validate_agg_attr_ref(agg_attrs, agg_obj): def validate_agg_collab_references(all_collborators, agg_obj, agg_attrs): - """ Iterate attributes of aggregator and collborator to capture the mismatched references. """ @@ -322,7 +318,7 @@ def validate_agg_collab_references(all_collborators, agg_obj, agg_attrs): agg_attr_id = id(getattr(agg_obj, attr)) for collab in all_collborators: collab_attr_id = id(getattr(collab, attr)) - if agg_attr_id == collab_attr_id: + if agg_attr_id is collab_attr_id: attr_ref_flag = True mis_matched_ref.get(collab).append(attr) @@ -338,26 +334,34 @@ def validate_agg_collab_references(all_collborators, agg_obj, agg_attrs): if __name__ == "__main__": - # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - ref_exception_list = [] + # Setup collaborators private attributes via callable function + collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] - # Setup collaborators with private attributes - collaborator_names = ["Portland", "Seattle"] # , 'Chandler', 'Bangalore'] - collaborators = [Collaborator(name=name) for name in collaborator_names] - collaborator.private_attributes = {} + def callable_to_initialize_collaborator_private_attributes(index): + return {"index": index + 1} + + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + index=idx, + ) + ) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") @@ -365,9 +369,6 @@ def validate_agg_collab_references(all_collborators, agg_obj, agg_attrs): testflow = TestFlowReference(checkpoint=True) testflow.runtime = local_runtime - for idx, collab in enumerate(collaborators): - collab.private_attributes = {"index": idx + 1} - for i in range(2): print(f"Starting round {i}...") testflow.run() diff --git a/tests/github/experimental/testflow_reference_with_exclude.py b/tests/github/experimental/testflow_reference_with_exclude.py index acc3f03759..0b5ffa93b7 100644 --- a/tests/github/experimental/testflow_reference_with_exclude.py +++ b/tests/github/experimental/testflow_reference_with_exclude.py @@ -46,8 +46,8 @@ class TestFlowReferenceWithExclude(FLSpec): """ - step_one_collab_attrs = {} - step_two_collab_attrs = {} + step_one_collab_attrs = [] + step_two_collab_attrs = [] all_ref_error_dict = {} @aggregator @@ -65,7 +65,6 @@ def start(self): @aggregator def test_create_agg_attr(self): - """ Create different types of objects """ @@ -86,52 +85,43 @@ def test_create_agg_attr(self): @collaborator def test_create_collab_attr(self): - """ Create different types of objects """ self.collab_attr_list_one = [1, 2, 3, 5, 6, 8] self.collab_attr_dict_one = {key: key for key in range(5)} - attr_collab_dict, collab_attr_list = create_collab_dict(self) - TestFlowReferenceWithExclude.step_one_collab_attrs.update( - attr_collab_dict - ) + TestFlowReferenceWithExclude.step_one_collab_attrs.append(self) if ( len(TestFlowReferenceWithExclude.step_one_collab_attrs) >= MIN_COLLECTION_COUNT ): - matched_ref_dict = find_match_ref_at_step( + collab_attr_list = filter_attrs(inspect.getmembers(self)) + matched_ref_dict = find_matched_references( collab_attr_list, TestFlowReferenceWithExclude.step_one_collab_attrs, ) validate_references(matched_ref_dict) - self.next( - self.test_create_more_collab_attr, exclude=["collab_attr_dict_one"] - ) + self.next(self.test_create_more_collab_attr, exclude=["collab_attr_dict_one"]) @collaborator def test_create_more_collab_attr(self): - """ Create different types of objects """ - self.collab_attr_list_two = [1, 2, 3, 5, 6, 8] self.collab_attr_dict_two = {key: key for key in range(5)} - attr_collab_dict, collab_attr_list = create_collab_dict(self) - TestFlowReferenceWithExclude.step_two_collab_attrs.update( - attr_collab_dict - ) + TestFlowReferenceWithExclude.step_two_collab_attrs.append(self) if ( len(TestFlowReferenceWithExclude.step_two_collab_attrs) >= MIN_COLLECTION_COUNT ): - matched_ref_dict = find_match_ref_at_step( + collab_attr_list = filter_attrs(inspect.getmembers(self)) + matched_ref_dict = find_matched_references( collab_attr_list, TestFlowReferenceWithExclude.step_two_collab_attrs, ) @@ -156,10 +146,7 @@ def join(self, inputs): f"\n{bcolors.UNDERLINE}Reference with exclude keyword test summary: {bcolors.ENDC}\n" ) - for ( - key, - val, - ) in TestFlowReferenceWithExclude.all_ref_error_dict.items(): + for val in TestFlowReferenceWithExclude.all_ref_error_dict.values(): all_shared_attr = all_shared_attr + ",".join(val) if all_shared_attr: @@ -167,9 +154,7 @@ def join(self, inputs): f"{bcolors.FAIL}...Test case failed for {all_shared_attr} {bcolors.ENDC}" ) else: - print( - f"{bcolors.OKGREEN}...Test case passed for all the attributes." - ) + print(f"{bcolors.OKGREEN}...Test case passed for all the attributes.") self.next(self.end) @aggregator @@ -186,8 +171,8 @@ def end(self): ) ) - TestFlowReferenceWithExclude.step_one_collab_attrs = {} - TestFlowReferenceWithExclude.step_two_collab_attrs = {} + TestFlowReferenceWithExclude.step_one_collab_attrs = [] + TestFlowReferenceWithExclude.step_two_collab_attrs = [] TestFlowReferenceWithExclude.all_ref_error_dict = {} @@ -205,33 +190,36 @@ def filter_attrs(attr_list): return valid_attrs -def find_matched_references(collab_attr_list, all_collborators): +def find_matched_references(collab_attr_list, all_collaborators): """ Iterate attributes of collborator and capture the duplicate reference + return: dict: { + 'Portland': ['failed attributes'], 'Seattle': [], + } """ matched_ref_dict = {} - previous_collaborator = "" - # Initialize dictionary with collborator as key and value as empty list to hold - # duplicated attr list - for collborator_name in all_collborators: - matched_ref_dict[collborator_name.input] = [] - - # Iterate the attributes and get duplicate attribute id - for attr in collab_attr_list: - attr_dict = {attr: []} - for collab in all_collborators: - attr_id = id(getattr(collab, attr)) - collaborator_name = collab.input - if attr_id not in attr_dict.get(attr): - attr_dict.get(attr).append(attr_id) - else: - # append the dict with collabartor as key and attrs as value having same reference - matched_ref_dict.get(collaborator_name).append(attr) - print( - f"{bcolors.FAIL} ... Reference test failed - {collaborator_name} sharing same " - + f"{attr} reference with {previous_collaborator} {bcolors.ENDC}" - ) - previous_collaborator = collaborator_name + for i in range(len(all_collaborators)): + matched_ref_dict[all_collaborators[i].input] = [] + + # For each attribute in the collaborator attribute list, check if any of the collaborator + # attributes are shared with another collaborator + for attr_name in collab_attr_list: + for i, curr_collab in enumerate(all_collaborators): + # Compare the current collaborator with the collaborator(s) that come(s) after it. + for next_collab in all_collaborators[i + 1:]: + # Check if both collaborators have the current attribute + if hasattr(curr_collab, attr_name) and hasattr(next_collab, attr_name): + # Check if both collaborators are sharing same reference + if getattr(curr_collab, attr_name) is getattr( + next_collab, attr_name + ): + matched_ref_dict[curr_collab.input].append(attr_name) + print( + f"{bcolors.FAIL} ... Reference test failed - {curr_collab.input} \ + sharing same " + + f"{attr_name} reference with {next_collab.input} {bcolors.ENDC}" + ) + return matched_ref_dict @@ -254,85 +242,33 @@ def validate_references(matched_ref_dict): ] = matched_ref_dict.get(collab) if not reference_flag: - print( - f"{bcolors.OKGREEN} Pass : Reference test passed {bcolors.ENDC}" - ) - - -def create_collab_dict(collab): - """ - saving the collaborator and its attributes to compare with other collaborator references. - return : dict ({ - 'Portland': {'collab_attr_dict_one': 140512653871680}, - 'Seattle': {'collab_attr_dict_one': 140512653871936} - }) - """ - attr_collab_dict = {} - collab_attr_list = filter_attrs(inspect.getmembers(collab)) - for attr in collab_attr_list: - attr_id = id(getattr(collab, attr)) - if attr_collab_dict.get(collab.input): - attr_collab_dict.get(collab.input)[attr] = attr_id - else: - attr_collab_dict[collab.input] = {} - attr_collab_dict.get(collab.input)[attr] = attr_id - return attr_collab_dict, collab_attr_list - - -def find_match_ref_at_step(collab_attr_list, all_collborators): - """ - Determines whether the current attributes are shared with - other participant attributes. If attributes are shared, - the test fails - """ - collab_names = all_collborators.keys() - matched_ref_dict = {} - for collborator_name in collab_names: - matched_ref_dict[collborator_name] = [] - - previous_collaborator = "" - for attr in collab_attr_list: - attr_dict = {attr: []} - for collborator_name in all_collborators.keys(): - attr_id = all_collborators[collborator_name][attr] - if attr_id not in attr_dict.get(attr): - attr_dict.get(attr).append(attr_id) - else: - matched_ref_dict.get(collborator_name).append(attr) - print( - f"{bcolors.FAIL} ... Reference test failed - {collborator_name} sharing same " - + f"{attr} reference with {previous_collaborator} {bcolors.ENDC}" - ) - - previous_collaborator = collborator_name - - return matched_ref_dict + print(f"{bcolors.OKGREEN} Pass : Reference test passed {bcolors.ENDC}") if __name__ == "__main__": - # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] - collaborators = [Collaborator(name=name) for name in collaborator_names] - collaborator.private_attributes = {} + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append(Collaborator(name=collaborator_name)) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") - testflow = TestFlowReferenceWithExclude(checkpoint=False) + testflow = TestFlowReferenceWithExclude(checkpoint=True) testflow.runtime = local_runtime for i in range(5): diff --git a/tests/github/experimental/testflow_reference_with_include.py b/tests/github/experimental/testflow_reference_with_include.py index 6e1ff76327..954b20ae5a 100644 --- a/tests/github/experimental/testflow_reference_with_include.py +++ b/tests/github/experimental/testflow_reference_with_include.py @@ -45,8 +45,8 @@ class TestFlowReferenceWithInclude(FLSpec): """ - step_one_collab_attrs = {} - step_two_collab_attrs = {} + step_one_collab_attrs = [] + step_two_collab_attrs = [] all_ref_error_dict = {} @aggregator @@ -64,7 +64,6 @@ def start(self): @aggregator def test_create_agg_attr(self): - """ Create different types of objects """ @@ -85,7 +84,6 @@ def test_create_agg_attr(self): @collaborator def test_create_collab_attr(self): - """ Modify the attirbutes of aggregator to validate the references. Create different types of objects. @@ -95,25 +93,20 @@ def test_create_collab_attr(self): self.collab_attr_dict_one = {key: key for key in range(5)} # append self attributes of collaborators - attr_collab_dict, collab_attr_list = create_collab_dict(self) - TestFlowReferenceWithInclude.step_one_collab_attrs.update( - attr_collab_dict - ) + TestFlowReferenceWithInclude.step_one_collab_attrs.append(self) if ( len(TestFlowReferenceWithInclude.step_one_collab_attrs) >= MIN_COLLECTION_COUNT ): - matched_ref_dict = find_match_ref_at_step( + collab_attr_list = filter_attrs(inspect.getmembers(self)) + matched_ref_dict = find_matched_references( collab_attr_list, TestFlowReferenceWithInclude.step_one_collab_attrs, ) validate_references(matched_ref_dict) - # must be tested with include functionality - self.next( - self.test_create_more_collab_attr, include=["collab_attr_dict_one"] - ) + self.next(self.test_create_more_collab_attr, include=["collab_attr_dict_one"]) @collaborator def test_create_more_collab_attr(self): @@ -124,16 +117,14 @@ def test_create_more_collab_attr(self): self.collab_attr_list_two = [1, 2, 3, 5, 6, 8] self.collab_attr_dict_two = {key: key for key in range(5)} - attr_collab_dict, collab_attr_list = create_collab_dict(self) - TestFlowReferenceWithInclude.step_two_collab_attrs.update( - attr_collab_dict - ) + TestFlowReferenceWithInclude.step_two_collab_attrs.append(self) if ( len(TestFlowReferenceWithInclude.step_two_collab_attrs) >= MIN_COLLECTION_COUNT ): - matched_ref_dict = find_match_ref_at_step( + collab_attr_list = filter_attrs(inspect.getmembers(self)) + matched_ref_dict = find_matched_references( collab_attr_list, TestFlowReferenceWithInclude.step_two_collab_attrs, ) @@ -154,16 +145,14 @@ def join(self, inputs): validate_references(matched_ref_dict) all_shared_attr = "" print(f"\n{bcolors.UNDERLINE}Reference test summary: {bcolors.ENDC}\n") - for key, val in TestFlowReferenceWithInclude.all_ref_error_dict.items(): + for val in TestFlowReferenceWithInclude.all_ref_error_dict.values(): all_shared_attr = all_shared_attr + ",".join(val) if all_shared_attr: print( f"{bcolors.FAIL}...Test case failed for {all_shared_attr} {bcolors.ENDC}" ) else: - print( - f"{bcolors.OKGREEN}...Test case passed for all the attributes." - ) + print(f"{bcolors.OKGREEN}...Test case passed for all the attributes.") self.next(self.end) @aggregator @@ -179,8 +168,8 @@ def end(self): ) ) - TestFlowReferenceWithInclude.step_one_collab_attrs = {} - TestFlowReferenceWithInclude.step_two_collab_attrs = {} + TestFlowReferenceWithInclude.step_one_collab_attrs = [] + TestFlowReferenceWithInclude.step_two_collab_attrs = [] TestFlowReferenceWithInclude.all_ref_error_dict = {} @@ -198,33 +187,36 @@ def filter_attrs(attr_list): return valid_attrs -def find_matched_references(collab_attr_list, all_collborators): +def find_matched_references(collab_attr_list, all_collaborators): """ Iterate attributes of collborator and capture the duplicate reference + return: dict: { + 'Portland': ['failed attributes'], 'Seattle': [], + } """ matched_ref_dict = {} - previous_collaborator = "" - # Initialize dictionary with collborator as key and value as empty list to hold - # duplicated attr list - for collborator_name in all_collborators: - matched_ref_dict[collborator_name.input] = [] - - # Iterate the attributes and get duplicate attribute id - for attr in collab_attr_list: - attr_dict = {attr: []} - for collab in all_collborators: - attr_id = id(getattr(collab, attr)) - collaborator_name = collab.input - if attr_id not in attr_dict.get(attr): - attr_dict.get(attr).append(attr_id) - else: - # append the dict with collabartor as key and attrs as value having same reference - matched_ref_dict.get(collaborator_name).append(attr) - print( - f"{bcolors.FAIL} ... Reference test failed - {collaborator_name} sharing same " - + f"{attr} reference with {previous_collaborator} {bcolors.ENDC}" - ) - previous_collaborator = collaborator_name + for i in range(len(all_collaborators)): + matched_ref_dict[all_collaborators[i].input] = [] + + # For each attribute in the collaborator attribute list, check if any of the collaborator + # attributes are shared with another collaborator + for attr_name in collab_attr_list: + for i, curr_collab in enumerate(all_collaborators): + # Compare the current collaborator with the collaborator(s) that come(s) after it. + for next_collab in all_collaborators[i + 1:]: + # Check if both collaborators have the current attribute + if hasattr(curr_collab, attr_name) and hasattr(next_collab, attr_name): + # Check if both collaborators are sharing same reference + if getattr(curr_collab, attr_name) is getattr( + next_collab, attr_name + ): + matched_ref_dict[curr_collab.input].append(attr_name) + print( + f"{bcolors.FAIL} ... Reference test failed - {curr_collab.input} \ + sharing same " + + f"{attr_name} reference with {next_collab.input} {bcolors.ENDC}" + ) + return matched_ref_dict @@ -247,81 +239,33 @@ def validate_references(matched_ref_dict): ] = matched_ref_dict.get(collab) if not reference_flag: - print( - f"{bcolors.OKGREEN} Pass : Reference test passed {bcolors.ENDC}" - ) - - -def create_collab_dict(collab): - """ - saving the collaborator and its attributes to compare with other collaborator refences. - return : dict ({ - 'Portland': {'collab_attr_dict_one': 140512653871680}, - 'Seattle': {'collab_attr_dict_one': 140512653871936} - }) - """ - attr_collab_dict = {} - collab_attr_list = filter_attrs(inspect.getmembers(collab)) - for attr in collab_attr_list: - attr_id = id(getattr(collab, attr)) - if attr_collab_dict.get(collab.input): - attr_collab_dict.get(collab.input)[attr] = attr_id - else: - attr_collab_dict[collab.input] = {} - attr_collab_dict.get(collab.input)[attr] = attr_id - return attr_collab_dict, collab_attr_list - - -def find_match_ref_at_step(collab_attr_list, all_collborators): - collab_names = all_collborators.keys() - - matched_ref_dict = {} - for collborator_name in collab_names: - matched_ref_dict[collborator_name] = [] - - previous_collaborator = "" - for attr in collab_attr_list: - attr_dict = {attr: []} - for collborator_name in all_collborators.keys(): - attr_id = all_collborators[collborator_name][attr] - if attr_id not in attr_dict.get(attr): - attr_dict.get(attr).append(attr_id) - else: - matched_ref_dict.get(collborator_name).append(attr) - print( - f"{bcolors.FAIL} ... Reference test failed - {collborator_name} sharing same " - + f"{attr} reference with {previous_collaborator} {bcolors.ENDC}" - ) - - previous_collaborator = collborator_name - - return matched_ref_dict + print(f"{bcolors.OKGREEN} Pass : Reference test passed {bcolors.ENDC}") if __name__ == "__main__": - # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} - # Setup collaborators with private attributes + # Setup collaborators collaborator_names = ["Portland", "Seattle", "Chandler", "Bangalore"] - collaborators = [Collaborator(name=name) for name in collaborator_names] - collaborator.private_attributes = {} + collaborators = [] + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append(Collaborator(name=collaborator_name)) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) print(f"Local runtime collaborators = {local_runtime.collaborators}") - testflow = TestFlowReferenceWithInclude(checkpoint=False) + testflow = TestFlowReferenceWithInclude(checkpoint=True) testflow.runtime = local_runtime for i in range(5): diff --git a/tests/github/experimental/testflow_subset_of_collaborators.py b/tests/github/experimental/testflow_subset_of_collaborators.py index a4081b43ee..12fea10a92 100644 --- a/tests/github/experimental/testflow_subset_of_collaborators.py +++ b/tests/github/experimental/testflow_subset_of_collaborators.py @@ -47,9 +47,7 @@ def start(self): self.collaborators = self.runtime.collaborators # select subset of collaborators - self.subset_collabrators = self.collaborators[ - : random.choice(self.random_ints) - ] + self.subset_collabrators = self.collaborators[: random.choice(self.random_ints)] print( f"... Executing flow for {len(self.subset_collabrators)} collaborators out of Total: " @@ -84,15 +82,14 @@ def end(self): End of the flow """ - print( - f"End of the test case {TestFlowSubsetCollaborators.__name__} reached." - ) + print(f"End of the test case {TestFlowSubsetCollaborators.__name__} reached.") if __name__ == "__main__": + # Setup participants aggregator = Aggregator() - aggregator.private_attributes = {} + # Setup collaborators private attributes via callable function collaborator_names = [ "Portland", "Seattle", @@ -103,26 +100,31 @@ def end(self): "London", "New York", ] + + def callable_to_initialize_collaborator_private_attributes(collab_name): + return {"name": collab_name} + collaborators = [] - for name in collaborator_names: - temp_collab_obj = Collaborator(name=name) - temp_collab_obj.private_attributes = {"name": name} - collaborators.append(temp_collab_obj) - del temp_collab_obj + for idx, collaborator_name in enumerate(collaborator_names): + collaborators.append( + Collaborator( + name=collaborator_name, + private_attributes_callable=callable_to_initialize_collaborator_private_attributes, + collab_name=collaborator_name, + ) + ) local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators + aggregator=aggregator, + collaborators=collaborators, ) - if len(sys.argv) > 1: - if sys.argv[1] == 'ray': + if sys.argv[1] == "ray": local_runtime = LocalRuntime( - aggregator=aggregator, collaborators=collaborators, backend='ray' + aggregator=aggregator, collaborators=collaborators, backend="ray" ) - random_ints = random.sample( - range(1, len(collaborators) + 1), len(collaborators) - ) + random_ints = random.sample(range(1, len(collaborators) + 1), len(collaborators)) tc_pass_fail = {"passed": [], "failed": []} for round_num in range(len(collaborators)): print(f"{bcolors.OKBLUE}Starting round {round_num}...{bcolors.ENDC}")