From 60c91952c13229e4fa60e9aae1477c401d3017a8 Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Wed, 4 Dec 2024 22:31:09 +0530 Subject: [PATCH] Simplify setup (#1187) * Simplify setup.py Signed-off-by: Shah, Karan * Remove tensorboardX and never-used log_metric code Signed-off-by: Shah, Karan * Test reducing requirements Signed-off-by: Shah, Karan * Revert "Remove tensorboardX and never-used log_metric code" and add fn calls Signed-off-by: Shah, Karan * Revert tb removal Signed-off-by: Shah, Karan * Disable tensorboard logging for gramine CI test Signed-off-by: Shah, Karan --------- Signed-off-by: Shah, Karan --- .../workflows/tr_docker_gramine_direct.yml | 5 + openfl-docker/__init__.py | 2 + openfl/component/aggregator/aggregator.py | 16 +++ .../interface/interactive_api/experiment.py | 4 +- openfl/plugins/__init__.py | 2 + .../plugins/frameworks_adapters/__init__.py | 2 + .../plugins/interface_serializer/__init__.py | 2 + .../processing_units_monitor/__init__.py | 2 + setup.py | 134 +++--------------- 9 files changed, 54 insertions(+), 115 deletions(-) create mode 100644 openfl-docker/__init__.py create mode 100644 openfl/plugins/__init__.py create mode 100644 openfl/plugins/frameworks_adapters/__init__.py create mode 100644 openfl/plugins/interface_serializer/__init__.py create mode 100644 openfl/plugins/processing_units_monitor/__init__.py diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index d8f7480ea1..be2afea5f9 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -31,6 +31,11 @@ jobs: fx workspace create --prefix example_workspace --template keras_cnn_mnist cd example_workspace fx plan initialize -a localhost + + # Disable tensorboard logging as multiprocessing is not supported in Gramine + # https://github.com/gramineproject/examples/issues/33 + sed -i 's/write_logs: true/write_logs: false/g' plan/plan.yaml + fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} - name: Create certificate authority for workspace diff --git a/openfl-docker/__init__.py b/openfl-docker/__init__.py new file mode 100644 index 0000000000..44936c23a2 --- /dev/null +++ b/openfl-docker/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/openfl/component/aggregator/aggregator.py b/openfl/component/aggregator/aggregator.py index 3128ef3e89..038a1e00d3 100644 --- a/openfl/component/aggregator/aggregator.py +++ b/openfl/component/aggregator/aggregator.py @@ -664,6 +664,14 @@ def send_local_task_results( } self.metric_queue.put(metrics) self.logger.metric("%s", str(metrics)) + if self.write_logs: + self.log_metric( + collaborator_name, + task_name, + tensor_key.tensor_name, + float(value), + round_number, + ) task_results.append(tensor_key) @@ -938,6 +946,14 @@ def _compute_validation_related_task_metrics(self, task_name): self.metric_queue.put(metrics) self.logger.metric("%s", metrics) + if self.write_logs: + self.log_metric( + "aggregator", + task_name, + tensor_key.tensor_name, + float(agg_results), + round_number, + ) # FIXME: Configurable logic for min/max criteria in saving best. if "validate_agg" in tags: diff --git a/openfl/interface/interactive_api/experiment.py b/openfl/interface/interactive_api/experiment.py index 111c6f1238..d8096df8ad 100644 --- a/openfl/interface/interactive_api/experiment.py +++ b/openfl/interface/interactive_api/experiment.py @@ -15,8 +15,6 @@ from shutil import copytree, ignore_patterns, make_archive from typing import Dict, Tuple -from tensorboardX import SummaryWriter - from openfl.component.assigner.tasks import Task, TrainTask, ValidateTask from openfl.federated import Plan from openfl.interface.aggregation_functions import AggregationFunction, WeightedAverage @@ -206,6 +204,8 @@ def stream_metrics(self, tensorboard_logs: bool = True) -> None: def write_tensorboard_metric(self, metric: dict) -> None: """Write metric callback.""" if not self.summary_writer: + from tensorboardX import SummaryWriter + self.summary_writer = SummaryWriter(f"./logs/{self.experiment_name}", flush_secs=5) self.summary_writer.add_scalar( diff --git a/openfl/plugins/__init__.py b/openfl/plugins/__init__.py new file mode 100644 index 0000000000..44936c23a2 --- /dev/null +++ b/openfl/plugins/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/openfl/plugins/frameworks_adapters/__init__.py b/openfl/plugins/frameworks_adapters/__init__.py new file mode 100644 index 0000000000..44936c23a2 --- /dev/null +++ b/openfl/plugins/frameworks_adapters/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/openfl/plugins/interface_serializer/__init__.py b/openfl/plugins/interface_serializer/__init__.py new file mode 100644 index 0000000000..44936c23a2 --- /dev/null +++ b/openfl/plugins/interface_serializer/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/openfl/plugins/processing_units_monitor/__init__.py b/openfl/plugins/processing_units_monitor/__init__.py new file mode 100644 index 0000000000..44936c23a2 --- /dev/null +++ b/openfl/plugins/processing_units_monitor/__init__.py @@ -0,0 +1,2 @@ +# Copyright 2020-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 diff --git a/setup.py b/setup.py index 4133fb69d0..bbdfa03b96 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,9 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 +"""Setup script.""" -"""This package includes dependencies of the openfl project.""" - -from setuptools import Command -from setuptools import setup +from setuptools import Command, find_packages, setup from setuptools.command.build_py import build_py -from setuptools.command.develop import develop class BuildPackageProtos(Command): @@ -61,113 +58,35 @@ def run(self): super().run() -class DevelopGRPC(develop): - """Command for develop installation.""" - - def __init__(self, dist): - """Create a sub-command to execute.""" - self.subcommand = BuildPackageProtos(dist) - super().__init__(dist) - - def run(self): - """Build GRPC modules before the default installation.""" - self.subcommand.run() - super().run() - - -with open('README.md', encoding='utf-8') as f: - long_description = f.read() - setup( name='openfl', version='1.6', - author='The OpenFL Team', + author='OpenFL Team', description='Federated Learning for the Edge', - long_description=long_description, + long_description=open("README.md", encoding="utf-8").read(), long_description_content_type='text/markdown', url='https://github.com/securefederatedai/openfl', - packages=[ - 'openfl', - 'openfl.component', - 'openfl.interface.aggregation_functions', - 'openfl.interface.aggregation_functions.core', - 'openfl.interface.aggregation_functions.experimental', - 'openfl.component.aggregator', - 'openfl.component.assigner', - 'openfl.component.collaborator', - 'openfl.component.director', - 'openfl.component.envoy', - 'openfl.component.straggler_handling_functions', - 'openfl.cryptography', - 'openfl.databases', - 'openfl.databases.utilities', - 'openfl.experimental', - 'openfl.experimental.workflow', - 'openfl.experimental.workflow.workspace_export', - 'openfl.experimental.workflow.federated', - 'openfl.experimental.workflow.federated.plan', - 'openfl.experimental.workflow.component', - 'openfl.experimental.workflow.component.aggregator', - 'openfl.experimental.workflow.component.collaborator', - 'openfl.experimental.workflow.interface.cli', - 'openfl.experimental.workflow.interface', - 'openfl.experimental.workflow.placement', - 'openfl.experimental.workflow.runtime', - 'openfl.experimental.workflow.protocols', - 'openfl.experimental.workflow.transport', - 'openfl.experimental.workflow.transport.grpc', - 'openfl.experimental.workflow.utilities', - 'openfl.federated', - 'openfl.federated.data', - 'openfl.federated.plan', - 'openfl.federated.task', - 'openfl.interface', - 'openfl.interface.interactive_api', - 'openfl.native', - 'openfl.pipelines', - 'openfl.plugins', - 'openfl.plugins.frameworks_adapters', - 'openfl.plugins.interface_serializer', - 'openfl.plugins.processing_units_monitor', - 'openfl.protocols', - 'openfl.transport', - 'openfl.transport.grpc', - 'openfl.utilities', - 'openfl.utilities.ca', - 'openfl.utilities.data_splitters', - 'openfl.utilities.fedcurv', - 'openfl.utilities.fedcurv.torch', - 'openfl.utilities.optimizers.keras', - 'openfl.utilities.optimizers.numpy', - 'openfl.utilities.optimizers.torch', - 'openfl-docker', - 'openfl-tutorials', - 'openfl-workspace', - ], + packages=find_packages(include=("openfl", "openfl.*", "openfl-docker", "openfl-workspace")), include_package_data=True, + setup_requires=['grpcio-tools>=1.56.2,<1.66.0'], # ensure it is in-sync with `install_requires` install_requires=[ - 'Click==8.1.7', - 'PyYAML>=5.4.1', - 'cloudpickle', - 'cryptography>=3.4.6', - 'docker', - 'dynaconf==3.2.6', - 'flatten_json', - 'grpcio>=1.56.2,<1.66.0', - 'ipykernel', - 'jupyterlab', + 'click', + 'psutil', + 'pyyaml', + 'rich', + 'dynaconf', + 'tqdm', 'numpy', + 'requests', + 'cloudpickle', + 'cryptography', 'pandas', - 'protobuf>=4.22,<6.0.0', - 'pyzmq<=26.2.0', - 'requests>=2.32.0', - 'rich', 'scikit-learn', - 'tensorboard', - 'tensorboardX>=2.6', - 'tqdm', + 'flatten_json', + 'tensorboardX', + 'protobuf>=4.22,<6.0.0', + 'grpcio>=1.56.2,<1.66.0', ], - setup_requires=['grpcio-tools>=1.56.2,<1.66.0'], python_requires='>=3.8, <3.12', project_urls={ 'Bug Tracker': 'https://github.com/securefederatedai/openfl/issues', @@ -176,31 +95,20 @@ def run(self): }, classifiers=[ 'Environment :: Console', - # How mature is this project? Common values are - # 3 - Alpha, 4 - Beta, 5 - Production/Stable 'Development Status :: 5 - Production/Stable', - # Indicate who your project is intended for - 'Intended Audience :: Developers', + 'Intended Audience :: Science/Research', 'Topic :: Scientific/Engineering :: Artificial Intelligence', - 'Topic :: Scientific/Engineering :: Image Recognition', 'Topic :: System :: Distributed Computing', - # Pick your license as you wish 'License :: OSI Approved :: Apache Software License', - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', - ], - entry_points={ - 'console_scripts': ['fx=openfl.interface.cli:entry'] - }, + entry_points={'console_scripts': ['fx=openfl.interface.cli:entry']}, cmdclass={ 'build_py': BuildPyGRPC, 'build_grpc': BuildPackageProtos, - 'develop': DevelopGRPC }, )