From 58eeac91f9247b84beb97725c1f9f0ce937e0ad8 Mon Sep 17 00:00:00 2001 From: Karan Shah Date: Mon, 2 Dec 2024 21:07:24 +0530 Subject: [PATCH] Migration to ruff (#1176) * Migrate to ruff tooling and config Signed-off-by: Shah, Karan * Code adaptations for new rules Signed-off-by: Shah, Karan * Silence selected C901 errors Signed-off-by: Shah, Karan * Reformat Signed-off-by: Shah, Karan --------- Signed-off-by: Shah, Karan --- CONTRIBUTING.md | 2 +- linters-requirements.txt | 4 +- openfl/__version__.py | 1 + openfl/component/aggregator/aggregator.py | 1 + openfl/component/collaborator/collaborator.py | 1 + openfl/component/director/director.py | 2 +- .../cutoff_time_based_straggler_handling.py | 1 + .../percentage_based_straggler_handling.py | 1 + openfl/cryptography/participant.py | 1 + .../component/aggregator/aggregator.py | 7 +- .../component/collaborator/collaborator.py | 5 +- .../workflow/federated/plan/plan.py | 1 + .../workflow/interface/cli/aggregator.py | 1 + .../workflow/interface/cli/cli_helper.py | 99 +-------------- .../workflow/interface/cli/collaborator.py | 3 +- .../workflow/interface/cli/plan.py | 1 + .../workflow/interface/cli/workspace.py | 4 +- .../workflow/interface/participants.py | 4 +- .../workflow/placement/placement.py | 4 +- .../workflow/protocols/interceptors.py | 1 + .../experimental/workflow/runtime/__init__.py | 2 +- .../workflow/runtime/federated_runtime.py | 5 +- .../workflow/runtime/local_runtime.py | 5 +- .../experimental/workflow/runtime/runtime.py | 2 +- .../workflow/transport/__init__.py | 1 + .../workflow/utilities/metaflow_utils.py | 8 +- .../workflow/utilities/runtime_utils.py | 4 +- openfl/experimental/workflow/utilities/ui.py | 4 +- .../workflow/workspace_export/export.py | 14 +-- openfl/federated/data/loader_gandlf.py | 1 + openfl/federated/plan/plan.py | 3 +- openfl/federated/task/fl_model.py | 3 +- openfl/federated/task/runner_keras.py | 1 + openfl/federated/task/runner_pt.py | 5 +- openfl/federated/task/runner_xgb.py | 32 +++-- .../aggregation_functions/core/interface.py | 1 + .../experimental/privileged_aggregation.py | 1 + .../aggregation_functions/fed_bagging.py | 3 +- .../fedcurv_weighted_average.py | 1 + openfl/interface/aggregator.py | 1 + openfl/interface/cli.py | 1 + openfl/interface/cli_helper.py | 118 +----------------- openfl/interface/collaborator.py | 38 +++--- openfl/interface/director.py | 1 + openfl/interface/envoy.py | 1 + openfl/interface/experimental.py | 1 + .../interface/interactive_api/experiment.py | 11 +- openfl/interface/model.py | 1 + openfl/interface/plan.py | 1 + openfl/interface/tutorial.py | 1 + openfl/interface/workspace.py | 12 +- openfl/native/fastestimator.py | 5 +- openfl/native/native.py | 3 +- openfl/pipelines/eden_pipeline.py | 2 +- openfl/pipelines/stc_pipeline.py | 1 + .../frameworks_adapters/flax_adapter.py | 3 +- .../frameworks_adapters/keras_adapter.py | 1 + .../frameworks_adapters/pytorch_adapter.py | 1 + openfl/protocols/interceptors.py | 1 + openfl/transport/grpc/aggregator_client.py | 9 +- openfl/transport/grpc/aggregator_server.py | 9 +- openfl/utilities/ca.py | 1 + openfl/utilities/ca/ca.py | 1 + openfl/utilities/data_splitters/__init__.py | 1 + .../utilities/data_splitters/data_splitter.py | 1 + openfl/utilities/fed_timer.py | 2 +- openfl/utilities/optimizers/keras/fedprox.py | 1 + .../optimizers/numpy/base_optimizer.py | 1 + openfl/utilities/optimizers/torch/__init__.py | 1 + openfl/utilities/workspace.py | 1 + pyproject.toml | 30 ++++- setup.cfg | 39 ------ shell/format.sh | 6 +- shell/lint.sh | 6 +- 74 files changed, 188 insertions(+), 365 deletions(-) delete mode 100644 setup.cfg diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6eeb6f5a48..a89798d1e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,7 @@ pip install -r linters-requirements.txt ## Code style -OpenFL uses [black](https://black.readthedocs.io/en/stable/), [isort](https://pycqa.github.io/isort/) and [precommit](https://pre-commit.com/) to format the code. +OpenFL uses [ruff](https://github.com/astral-sh/ruff) to lint/format code and [precommit](https://pre-commit.com/) checks. Run the following command at the **root** directory of the repo to format your code. diff --git a/linters-requirements.txt b/linters-requirements.txt index f5ea6cbf18..b4a2191f21 100644 --- a/linters-requirements.txt +++ b/linters-requirements.txt @@ -1,4 +1,2 @@ -black -flake8 -isort pre-commit +ruff diff --git a/openfl/__version__.py b/openfl/__version__.py index 713ad49b87..44cb4cc1db 100644 --- a/openfl/__version__.py +++ b/openfl/__version__.py @@ -3,4 +3,5 @@ """openfl version information.""" + __version__ = "1.6" diff --git a/openfl/component/aggregator/aggregator.py b/openfl/component/aggregator/aggregator.py index 805193cf77..3128ef3e89 100644 --- a/openfl/component/aggregator/aggregator.py +++ b/openfl/component/aggregator/aggregator.py @@ -3,6 +3,7 @@ """Aggregator module.""" + import queue import time from logging import getLogger diff --git a/openfl/component/collaborator/collaborator.py b/openfl/component/collaborator/collaborator.py index 3418afab73..b327cd9117 100644 --- a/openfl/component/collaborator/collaborator.py +++ b/openfl/component/collaborator/collaborator.py @@ -3,6 +3,7 @@ """Collaborator module.""" + from enum import Enum from logging import getLogger from time import sleep diff --git a/openfl/component/director/director.py b/openfl/component/director/director.py index f4ca3cc731..79ac85d961 100644 --- a/openfl/component/director/director.py +++ b/openfl/component/director/director.py @@ -368,7 +368,7 @@ def update_envoy_status( if not shard_info: raise ShardNotFoundError(f"Unknown shard {envoy_name}") - shard_info["is_online"]: True + shard_info["is_online"] = True shard_info["is_experiment_running"] = is_experiment_running shard_info["valid_duration"] = 2 * self.envoy_health_check_period shard_info["last_updated"] = time.time() diff --git a/openfl/component/straggler_handling_functions/cutoff_time_based_straggler_handling.py b/openfl/component/straggler_handling_functions/cutoff_time_based_straggler_handling.py index ca8e218f7c..c3d11ffa4b 100644 --- a/openfl/component/straggler_handling_functions/cutoff_time_based_straggler_handling.py +++ b/openfl/component/straggler_handling_functions/cutoff_time_based_straggler_handling.py @@ -3,6 +3,7 @@ """Cutoff time based Straggler Handling function.""" + import threading import time from logging import getLogger diff --git a/openfl/component/straggler_handling_functions/percentage_based_straggler_handling.py b/openfl/component/straggler_handling_functions/percentage_based_straggler_handling.py index e556ea291b..099ab7e870 100644 --- a/openfl/component/straggler_handling_functions/percentage_based_straggler_handling.py +++ b/openfl/component/straggler_handling_functions/percentage_based_straggler_handling.py @@ -3,6 +3,7 @@ """Percentage based Straggler Handling function.""" + from logging import getLogger from openfl.component.straggler_handling_functions.straggler_handling_function import ( diff --git a/openfl/cryptography/participant.py b/openfl/cryptography/participant.py index 6e18840ff9..f190f68714 100644 --- a/openfl/cryptography/participant.py +++ b/openfl/cryptography/participant.py @@ -3,6 +3,7 @@ """Cryptography participant utilities.""" + from typing import Tuple from cryptography import x509 diff --git a/openfl/experimental/workflow/component/aggregator/aggregator.py b/openfl/experimental/workflow/component/aggregator/aggregator.py index 1e4d7b48e4..4e034aa315 100644 --- a/openfl/experimental/workflow/component/aggregator/aggregator.py +++ b/openfl/experimental/workflow/component/aggregator/aggregator.py @@ -3,6 +3,7 @@ """Experimental Aggregator module.""" + import inspect import pickle import queue @@ -47,8 +48,8 @@ def __init__( rounds_to_train: int = 1, checkpoint: bool = False, private_attributes_callable: Callable = None, - private_attributes_kwargs: Dict = {}, - private_attributes: Dict = {}, + private_attributes_kwargs: Dict = None, + private_attributes: Dict = None, single_col_cert_common_name: str = None, log_metric_callback: Callable = None, **kwargs, @@ -232,7 +233,7 @@ def call_checkpoint(self, ctx: Any, f: Callable, stream_buffer: bytes = None) -> f = pickle.loads(f) if isinstance(stream_buffer, bytes): # Set stream buffer as function parameter - setattr(f.__func__, "_stream_buffer", pickle.loads(stream_buffer)) + f.__func__._stream_buffer = pickle.loads(stream_buffer) checkpoint(ctx, f) diff --git a/openfl/experimental/workflow/component/collaborator/collaborator.py b/openfl/experimental/workflow/component/collaborator/collaborator.py index d8e8a7fcd7..a683bbfeb7 100644 --- a/openfl/experimental/workflow/component/collaborator/collaborator.py +++ b/openfl/experimental/workflow/component/collaborator/collaborator.py @@ -3,6 +3,7 @@ """Experimental Collaborator module.""" + import pickle import time from logging import getLogger @@ -35,8 +36,8 @@ def __init__( federation_uuid: str, client: Any, private_attributes_callable: Any = None, - private_attributes_kwargs: Dict = {}, - private_attributes: Dict = {}, + private_attributes_kwargs: Dict = None, + private_attributes: Dict = None, **kwargs, ) -> None: self.name = collaborator_name diff --git a/openfl/experimental/workflow/federated/plan/plan.py b/openfl/experimental/workflow/federated/plan/plan.py index 22f2e2e2cd..6df27d8b1c 100644 --- a/openfl/experimental/workflow/federated/plan/plan.py +++ b/openfl/experimental/workflow/federated/plan/plan.py @@ -3,6 +3,7 @@ """Plan module.""" + import inspect import os from hashlib import sha384 diff --git a/openfl/experimental/workflow/interface/cli/aggregator.py b/openfl/experimental/workflow/interface/cli/aggregator.py index 2af60af020..2c458fa2fe 100644 --- a/openfl/experimental/workflow/interface/cli/aggregator.py +++ b/openfl/experimental/workflow/interface/cli/aggregator.py @@ -3,6 +3,7 @@ """Aggregator module.""" + import os import sys import threading diff --git a/openfl/experimental/workflow/interface/cli/cli_helper.py b/openfl/experimental/workflow/interface/cli/cli_helper.py index 65692d6c22..e45b3cdd09 100644 --- a/openfl/experimental/workflow/interface/cli/cli_helper.py +++ b/openfl/experimental/workflow/interface/cli/cli_helper.py @@ -3,11 +3,11 @@ """Module with auxiliary CLI helper functions.""" + import os import re -import shutil from itertools import islice -from os import environ, stat +from os import environ from pathlib import Path from sys import argv @@ -31,20 +31,6 @@ def pretty(o): echo(style(f"{k:<{m}} : ", fg="blue") + style(f"{v}", fg="cyan")) -def tree(path): - """Print current directory file tree.""" - echo(f"+ {path}") - - for path in sorted(path.rglob("*")): - depth = len(path.relative_to(path).parts) - space = " " * depth - - if path.is_file(): - echo(f"{space}f {path.name}") - else: - echo(f"{space}d {path.name}") - - def print_tree( dir_path: Path, level: int = -1, @@ -91,87 +77,6 @@ def inner(dir_path: Path, prefix: str = "", level=-1): echo(f"\n{directories} directories" + (f", {files} files" if files else "")) -def copytree( - src, - dst, - symlinks=False, - ignore=None, - ignore_dangling_symlinks=False, - dirs_exist_ok=False, -): - """From Python 3.8 'shutil' which include 'dirs_exist_ok' option.""" - - with os.scandir(src) as itr: - entries = list(itr) - - copy_function = shutil.copy2 - - def _copytree(): - if ignore is not None: - ignored_names = ignore(os.fspath(src), [x.name for x in entries]) - else: - ignored_names = set() - - os.makedirs(dst, exist_ok=dirs_exist_ok) - errors = [] - use_srcentry = copy_function is shutil.copy2 or copy_function is shutil.copy - - for srcentry in entries: - if srcentry.name in ignored_names: - continue - srcname = os.path.join(src, srcentry.name) - dstname = os.path.join(dst, srcentry.name) - srcobj = srcentry if use_srcentry else srcname - try: - is_symlink = srcentry.is_symlink() - if is_symlink and os.name == "nt": - lstat = srcentry.stat(follow_symlinks=False) - if lstat.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT: - is_symlink = False - if is_symlink: - linkto = os.readlink(srcname) - if symlinks: - os.symlink(linkto, dstname) - shutil.copystat(srcobj, dstname, follow_symlinks=not symlinks) - else: - if not os.path.exists(linkto) and ignore_dangling_symlinks: - continue - if srcentry.is_dir(): - copytree( - srcobj, - dstname, - symlinks, - ignore, - dirs_exist_ok=dirs_exist_ok, - ) - else: - copy_function(srcobj, dstname) - elif srcentry.is_dir(): - copytree( - srcobj, - dstname, - symlinks, - ignore, - dirs_exist_ok=dirs_exist_ok, - ) - else: - copy_function(srcobj, dstname) - except OSError as why: - errors.append((srcname, dstname, str(why))) - except Exception as err: - errors.extend(err.args[0]) - try: - shutil.copystat(src, dst) - except OSError as why: - if getattr(why, "winerror", None) is None: - errors.append((src, dst, str(why))) - if errors: - raise Exception(errors) - return dst - - return _copytree() - - def get_workspace_parameter(name): """Get a parameter from the workspace config file (.workspace).""" # Update the .workspace file to show the current workspace plan diff --git a/openfl/experimental/workflow/interface/cli/collaborator.py b/openfl/experimental/workflow/interface/cli/collaborator.py index 7398f01536..fe0cb32940 100644 --- a/openfl/experimental/workflow/interface/cli/collaborator.py +++ b/openfl/experimental/workflow/interface/cli/collaborator.py @@ -3,6 +3,7 @@ """Collaborator module.""" + import os import sys from glob import glob @@ -246,7 +247,7 @@ def certify_(collaborator_name, silent, request_pkg, import_): certify(collaborator_name, silent, request_pkg, import_) -def certify(collaborator_name, silent, request_pkg=None, import_=False): +def certify(collaborator_name, silent, request_pkg=None, import_=False): # noqa C901 """Sign/certify collaborator certificate key pair.""" common_name = f"{collaborator_name}" diff --git a/openfl/experimental/workflow/interface/cli/plan.py b/openfl/experimental/workflow/interface/cli/plan.py index 532350aa92..9f40db67de 100644 --- a/openfl/experimental/workflow/interface/cli/plan.py +++ b/openfl/experimental/workflow/interface/cli/plan.py @@ -3,6 +3,7 @@ """Plan module.""" + import sys from logging import getLogger from pathlib import Path diff --git a/openfl/experimental/workflow/interface/cli/workspace.py b/openfl/experimental/workflow/interface/cli/workspace.py index b2f7f54a73..1d65d0dde0 100644 --- a/openfl/experimental/workflow/interface/cli/workspace.py +++ b/openfl/experimental/workflow/interface/cli/workspace.py @@ -3,6 +3,7 @@ """Workspace module.""" + import os import sys from hashlib import sha256 @@ -16,9 +17,8 @@ from tempfile import mkdtemp from typing import Tuple -from click import Choice +from click import Choice, confirm, echo, group, option, pass_context, style from click import Path as ClickPath -from click import confirm, echo, group, option, pass_context, style from cryptography.hazmat.primitives import serialization from openfl.cryptography.ca import generate_root_cert, generate_signing_csr, sign_certificate diff --git a/openfl/experimental/workflow/interface/participants.py b/openfl/experimental/workflow/interface/participants.py index 618f40161d..5c9c9d594f 100644 --- a/openfl/experimental/workflow/interface/participants.py +++ b/openfl/experimental/workflow/interface/participants.py @@ -123,7 +123,7 @@ def __init__( private_attributes_callable: Callable = None, num_cpus: int = 0, num_gpus: int = 0.0, - **kwargs + **kwargs, ): """Initializes the Collaborator object. @@ -190,7 +190,7 @@ def __init__( private_attributes_callable: Callable = None, num_cpus: int = 0, num_gpus: int = 0.0, - **kwargs + **kwargs, ): """Initializes the Aggregator object. diff --git a/openfl/experimental/workflow/placement/placement.py b/openfl/experimental/workflow/placement/placement.py index 67458a0fca..348716e24f 100644 --- a/openfl/experimental/workflow/placement/placement.py +++ b/openfl/experimental/workflow/placement/placement.py @@ -41,7 +41,7 @@ def wrapper(*args, **kwargs): print(f"\nCalling {f.__name__}") with RedirectStdStreamContext() as context_stream: # context_stream capture stdout and stderr for the function f.__name__ - setattr(wrapper, "_stream_buffer", context_stream) + wrapper._stream_buffer = context_stream f(*args, **kwargs) return wrapper @@ -92,7 +92,7 @@ def wrapper(*args, **kwargs): print(f"\nCalling {f.__name__}") with RedirectStdStreamContext() as context_stream: # context_stream capture stdout and stderr for the function f.__name__ - setattr(wrapper, "_stream_buffer", context_stream) + wrapper._stream_buffer = context_stream f(*args, **kwargs) return wrapper diff --git a/openfl/experimental/workflow/protocols/interceptors.py b/openfl/experimental/workflow/protocols/interceptors.py index 3549529da6..e6a9b57b3f 100644 --- a/openfl/experimental/workflow/protocols/interceptors.py +++ b/openfl/experimental/workflow/protocols/interceptors.py @@ -3,6 +3,7 @@ """gRPC interceptors module.""" + import collections import grpc diff --git a/openfl/experimental/workflow/runtime/__init__.py b/openfl/experimental/workflow/runtime/__init__.py index 991792046e..5d13960b27 100644 --- a/openfl/experimental/workflow/runtime/__init__.py +++ b/openfl/experimental/workflow/runtime/__init__.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 -""" openfl.experimental.workflow.runtime package Runtime class.""" +"""openfl.experimental.workflow.runtime package Runtime class.""" from openfl.experimental.workflow.runtime.federated_runtime import FederatedRuntime from openfl.experimental.workflow.runtime.local_runtime import LocalRuntime diff --git a/openfl/experimental/workflow/runtime/federated_runtime.py b/openfl/experimental/workflow/runtime/federated_runtime.py index d07d477580..9684fad404 100644 --- a/openfl/experimental/workflow/runtime/federated_runtime.py +++ b/openfl/experimental/workflow/runtime/federated_runtime.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 -""" openfl.experimental.workflow.runtime package LocalRuntime class.""" +"""openfl.experimental.workflow.runtime package LocalRuntime class.""" from __future__ import annotations @@ -11,8 +11,7 @@ from openfl.experimental.workflow.runtime.runtime import Runtime if TYPE_CHECKING: - from openfl.experimental.workflow.interface import Aggregator - from openfl.experimental.workflow.interface import Collaborator + from openfl.experimental.workflow.interface import Aggregator, Collaborator from typing import List, Type diff --git a/openfl/experimental/workflow/runtime/local_runtime.py b/openfl/experimental/workflow/runtime/local_runtime.py index 978a684d65..7a7aa3f7a2 100644 --- a/openfl/experimental/workflow/runtime/local_runtime.py +++ b/openfl/experimental/workflow/runtime/local_runtime.py @@ -2,7 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 -""" openfl.experimental.workflow.runtime package LocalRuntime class.""" +"""openfl.experimental.workflow.runtime package LocalRuntime class.""" + from __future__ import annotations import gc @@ -420,7 +421,7 @@ def __get_aggregator_object(self, aggregator: Type[Aggregator]) -> Any: ) interface_module = importlib.import_module("openfl.experimental.workflow.interface") - aggregator_class = getattr(interface_module, "Aggregator") + aggregator_class = interface_module.Aggregator aggregator_actor = ray.remote(aggregator_class).options( num_cpus=agg_cpus, num_gpus=agg_gpus diff --git a/openfl/experimental/workflow/runtime/runtime.py b/openfl/experimental/workflow/runtime/runtime.py index 3cffeccafc..317c26ede6 100644 --- a/openfl/experimental/workflow/runtime/runtime.py +++ b/openfl/experimental/workflow/runtime/runtime.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 -""" openfl.experimental.workflow.runtime module Runtime class.""" +"""openfl.experimental.workflow.runtime module Runtime class.""" from typing import Callable, List diff --git a/openfl/experimental/workflow/transport/__init__.py b/openfl/experimental/workflow/transport/__init__.py index 83c441b84f..6a8abc0d3f 100644 --- a/openfl/experimental/workflow/transport/__init__.py +++ b/openfl/experimental/workflow/transport/__init__.py @@ -3,4 +3,5 @@ """openfl.experimental.workflow.transport package.""" + from openfl.experimental.workflow.transport.grpc import AggregatorGRPCClient, AggregatorGRPCServer diff --git a/openfl/experimental/workflow/utilities/metaflow_utils.py b/openfl/experimental/workflow/utilities/metaflow_utils.py index 21f567a88f..ddb08f1905 100644 --- a/openfl/experimental/workflow/utilities/metaflow_utils.py +++ b/openfl/experimental/workflow/utilities/metaflow_utils.py @@ -65,9 +65,7 @@ def __init__(self, name): self.name = name def __enter__(self): - lock_id = hashlib.new( - "sha256", self.name.encode("utf8"), usedforsecurity=False - ).hexdigest() # nosec + lock_id = hashlib.new("sha256", self.name.encode("utf8"), usedforsecurity=False).hexdigest() # nosec # Using SHA-256 to address security warning self.fp = open(f"/tmp/.lock-{lock_id}.lck", "wb") fcntl.flock(self.fp.fileno(), fcntl.LOCK_EX) @@ -158,7 +156,7 @@ def __init__(self, func_ast, decos, doc): # these attributes are populated by _postprocess self.is_inside_foreach = False - def _parse(self, func_ast): + def _parse(self, func_ast): # noqa: C901 self.num_args = len(func_ast.args.args) tail = func_ast.body[-1] @@ -319,7 +317,7 @@ def __init__( @only_if_not_done @require_mode("w") - def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): + def save_artifacts(self, artifacts_iter, force_v4=False, len_hint=0): # noqa: C901 """Saves Metaflow Artifacts (Python objects) to the datastore and stores any relevant metadata needed to retrieve them. diff --git a/openfl/experimental/workflow/utilities/runtime_utils.py b/openfl/experimental/workflow/utilities/runtime_utils.py index bd4ce92cb0..1aba29605a 100644 --- a/openfl/experimental/workflow/utilities/runtime_utils.py +++ b/openfl/experimental/workflow/utilities/runtime_utils.py @@ -69,7 +69,7 @@ def artifacts_iter(): return artifacts_iter, cls_attrs -def filter_attributes(ctx, f, **kwargs): +def filter_attributes(ctx, f, **kwargs): # noqa: C901 """Filters out attributes from the next task in the flow based on inclusion or exclusion. @@ -176,7 +176,7 @@ def check_resource_allocation(num_gpus, each_participant_gpu_usage): # buffer to cycle though since need_assigned will change sizes as we # assign participants current_dict = need_assigned.copy() - for i, (participant_name, participant_gpu_usage) in enumerate(current_dict.items()): + for participant_name, participant_gpu_usage in current_dict.items(): if gpu == 0: break if gpu < participant_gpu_usage: diff --git a/openfl/experimental/workflow/utilities/ui.py b/openfl/experimental/workflow/utilities/ui.py index 1ac59d1f93..d9b0c19207 100644 --- a/openfl/experimental/workflow/utilities/ui.py +++ b/openfl/experimental/workflow/utilities/ui.py @@ -27,7 +27,7 @@ def __init__( flow_obj, run_id, show_html=False, - ds_root=f"{Path.home()}/.metaflow", + ds_root=None, ): """Initializes the InspectFlow with a flow object, run ID, an optional flag to show the UI in a web browser, and an optional root directory @@ -41,7 +41,7 @@ def __init__( ds_root (str, optional): The root directory for the data store. Defaults to "~/.metaflow". """ - self.ds_root = ds_root + self.ds_root = ds_root or f"{Path.home()}/.metaflow" self.show_html = show_html self.run_id = run_id self.flow_name = flow_obj.__class__.__name__ diff --git a/openfl/experimental/workflow/workspace_export/export.py b/openfl/experimental/workflow/workspace_export/export.py index e0cc893f17..7df2a7684b 100644 --- a/openfl/experimental/workflow/workspace_export/export.py +++ b/openfl/experimental/workflow/workspace_export/export.py @@ -3,6 +3,7 @@ """Workspace Builder module.""" + import ast import importlib import inspect @@ -175,7 +176,7 @@ def __get_class_name_and_sourcecode_from_parent_class(self, parent_class): return None, None - def __extract_class_initializing_args(self, class_name): + def __extract_class_initializing_args(self, class_name): # noqa: C901 """Provided name of the class returns expected arguments and it's values in form of dictionary.""" instantiation_args = {"args": {}, "kwargs": {}} @@ -294,9 +295,7 @@ def generate_plan_yaml(self): """ Generates plan.yaml """ - flspec = getattr( - importlib.import_module("openfl.experimental.workflow.interface"), "FLSpec" - ) + flspec = importlib.import_module("openfl.experimental.workflow.interface").FLSpec # Get flow classname _, self.flow_class_name = self.__get_class_name_and_sourcecode_from_parent_class(flspec) # Get expected arguments of flow class @@ -335,7 +334,7 @@ def update_dictionary(args: dict, data: dict, dtype: str = "args"): self.__write_yaml(plan, data) - def generate_data_yaml(self): + def generate_data_yaml(self): # noqa: C901 """Generates data.yaml.""" # Import python script if not already if not hasattr(self, "exported_script_module"): @@ -343,10 +342,7 @@ def generate_data_yaml(self): # If flow classname is not yet found if not hasattr(self, "flow_class_name"): - flspec = getattr( - importlib.import_module("openfl.experimental.workflow.interface"), - "FLSpec", - ) + flspec = importlib.import_module("openfl.experimental.workflow.interface").FLSpec _, self.flow_class_name = self.__get_class_name_and_sourcecode_from_parent_class(flspec) # Import flow class diff --git a/openfl/federated/data/loader_gandlf.py b/openfl/federated/data/loader_gandlf.py index 648ebe2930..0dda8e3a46 100644 --- a/openfl/federated/data/loader_gandlf.py +++ b/openfl/federated/data/loader_gandlf.py @@ -3,6 +3,7 @@ """PyTorchDataLoader module.""" + from openfl.federated.data.loader import DataLoader diff --git a/openfl/federated/plan/plan.py b/openfl/federated/plan/plan.py index 455b0b1414..34c50a4d1e 100644 --- a/openfl/federated/plan/plan.py +++ b/openfl/federated/plan/plan.py @@ -3,6 +3,7 @@ """Plan module.""" + from hashlib import sha384 from importlib import import_module from logging import getLogger @@ -97,7 +98,7 @@ def ignore_aliases(self, data): yaml_path.write_text(dump(config)) @staticmethod - def parse( + def parse( # noqa: C901 plan_config_path: Path, cols_config_path: Path = None, data_config_path: Path = None, diff --git a/openfl/federated/task/fl_model.py b/openfl/federated/task/fl_model.py index 110c91c5bc..70bc537324 100644 --- a/openfl/federated/task/fl_model.py +++ b/openfl/federated/task/fl_model.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """FederatedModel module.""" + import inspect from openfl.federated.task.runner import TaskRunner @@ -129,7 +130,7 @@ def setup(self, num_collaborators, **kwargs): optimizer=self.lambda_opt, loss_fn=self.loss_fn, data_loader=data_slice, - **kwargs + **kwargs, ) for data_slice in self.data_loader.split(num_collaborators) ] diff --git a/openfl/federated/task/runner_keras.py b/openfl/federated/task/runner_keras.py index df56fd97e1..b33b20fde5 100644 --- a/openfl/federated/task/runner_keras.py +++ b/openfl/federated/task/runner_keras.py @@ -7,6 +7,7 @@ You may copy this file as the starting point of your own keras model. """ + from warnings import catch_warnings, simplefilter import numpy as np diff --git a/openfl/federated/task/runner_pt.py b/openfl/federated/task/runner_pt.py index 2adfe9d5bb..30b56a4120 100644 --- a/openfl/federated/task/runner_pt.py +++ b/openfl/federated/task/runner_pt.py @@ -504,8 +504,9 @@ def validate_(self, validation_dataloader: Iterator[Tuple[np.ndarray, np.ndarray for data, target in validation_dataloader: samples = target.shape[0] total_samples += samples - data, target = torch.tensor(data).to(self.device), torch.tensor(target).to( - self.device, dtype=torch.int64 + data, target = ( + torch.tensor(data).to(self.device), + torch.tensor(target).to(self.device, dtype=torch.int64), ) output = self(data) # get the index of the max log-probability diff --git a/openfl/federated/task/runner_xgb.py b/openfl/federated/task/runner_xgb.py index ae44210ce2..a5f5101b2e 100644 --- a/openfl/federated/task/runner_xgb.py +++ b/openfl/federated/task/runner_xgb.py @@ -48,7 +48,8 @@ def __init__(self, **kwargs): Attributes: global_model (xgb.Booster): The global XGBoost model. - required_tensorkeys_for_function (dict): A dictionary to store required tensor keys for each function. + required_tensorkeys_for_function (dict): A dictionary to store required tensor keys + for each function. """ super().__init__(**kwargs) self.global_model = None @@ -58,11 +59,13 @@ def rebuild_model(self, input_tensor_dict): """ Rebuilds the model using the provided input tensor dictionary. - This method checks if the 'local_tree' key in the input tensor dictionary is either a non-empty numpy array - If this condition is met, it updates the internal tensor dictionary with the provided input. + This method checks if the 'local_tree' key in the input tensor dictionary is either a + non-empty numpy array. If this condition is met, it updates the internal tensor dictionary + with the provided input. Parameters: - input_tensor_dict (dict): A dictionary containing tensor data. It must include the key 'local_tree' + input_tensor_dict (dict): A dictionary containing tensor data. + It must include the key 'local_tree' Returns: None @@ -90,11 +93,13 @@ def validate_task(self, col_name, round_num, input_tensor_dict, **kwargs): """ data = self.data_loader.get_valid_dmatrix() - # during agg validation, self.bst will still be None. during local validation, it will have a value - no need to rebuild + # during agg validation, self.bst will still be None. during local validation, + # it will have a value - no need to rebuild if self.bst is None: self.rebuild_model(input_tensor_dict) - # if self.bst is still None after rebuilding, then there was no initial global model, so set metric to 0 + # if self.bst is still None after rebuilding, then there was no initial global model, so + # set metric to 0 if self.bst is None: # for first round agg validation, there is no model so set metric to 0 # TODO: this is not robust, especially if using a loss metric @@ -188,16 +193,18 @@ def get_tensor_dict(self, with_opt_vars=False): """ Retrieves the tensor dictionary containing the model's tree structure. - This method returns a dictionary with the key 'local_tree', which contains the model's tree structure as a numpy array. - If the model has not been initialized (`self.bst` is None), it returns an empty numpy array. - If the global model is not set or is empty, it returns the entire model as a numpy array. - Otherwise, it returns only the trees added in the latest training session. + This method returns a dictionary with the key 'local_tree', which contains the model's tree + structure as a numpy array. If the model has not been initialized (`self.bst` is None), it + returns an empty numpy array. If the global model is not set or is empty, it returns the + entire model as a numpy array. Otherwise, it returns only the trees added in the latest + training session. Parameters: with_opt_vars (bool): N/A for XGBoost (Default=False). Returns: - dict: A dictionary with the key 'local_tree' containing the model's tree structure as a numpy array. + dict: A dictionary with the key 'local_tree' containing the model's tree structure as a + numpy array. """ if self.bst is None: @@ -377,7 +384,8 @@ def validate_(self, data) -> Metric: Validate the XGBoost model. Args: - validation_dataloader (dict): A dictionary containing the validation data with keys 'dmatrix' and 'labels'. + validation_dataloader (dict): A dictionary containing the validation data with keys + 'dmatrix' and 'labels'. Returns: Metric: A Metric object containing the validation accuracy. diff --git a/openfl/interface/aggregation_functions/core/interface.py b/openfl/interface/aggregation_functions/core/interface.py index c11fd62621..10f91ace3a 100644 --- a/openfl/interface/aggregation_functions/core/interface.py +++ b/openfl/interface/aggregation_functions/core/interface.py @@ -3,6 +3,7 @@ """Aggregation function interface module.""" + from abc import abstractmethod from typing import Iterator, List, Tuple diff --git a/openfl/interface/aggregation_functions/experimental/privileged_aggregation.py b/openfl/interface/aggregation_functions/experimental/privileged_aggregation.py index 00b8bf7625..0e8beacebc 100644 --- a/openfl/interface/aggregation_functions/experimental/privileged_aggregation.py +++ b/openfl/interface/aggregation_functions/experimental/privileged_aggregation.py @@ -3,6 +3,7 @@ """Aggregation function interface module.""" + from abc import abstractmethod from typing import List, Tuple diff --git a/openfl/interface/aggregation_functions/fed_bagging.py b/openfl/interface/aggregation_functions/fed_bagging.py index 2e42072c66..2f09481179 100644 --- a/openfl/interface/aggregation_functions/fed_bagging.py +++ b/openfl/interface/aggregation_functions/fed_bagging.py @@ -35,7 +35,8 @@ def append_trees(global_model, local_trees): Parameters: global_model (dict): A dictionary representing the global model. - local_trees (list): A list of dictionaries representing the local trees to be appended to the global model. + local_trees (list): A list of dictionaries representing the local trees to be appended to the + global model. Returns: dict: The updated global model with the local trees appended. diff --git a/openfl/interface/aggregation_functions/fedcurv_weighted_average.py b/openfl/interface/aggregation_functions/fedcurv_weighted_average.py index ba87da9c11..7fb0ffe992 100644 --- a/openfl/interface/aggregation_functions/fedcurv_weighted_average.py +++ b/openfl/interface/aggregation_functions/fedcurv_weighted_average.py @@ -3,6 +3,7 @@ """FedCurv Aggregation function module.""" + import numpy as np from openfl.interface.aggregation_functions.weighted_average import WeightedAverage diff --git a/openfl/interface/aggregator.py b/openfl/interface/aggregator.py index 60bd874ee9..80ce56e32e 100644 --- a/openfl/interface/aggregator.py +++ b/openfl/interface/aggregator.py @@ -3,6 +3,7 @@ """Aggregator module.""" + import sys from logging import getLogger from pathlib import Path diff --git a/openfl/interface/cli.py b/openfl/interface/cli.py index e497f47e9e..314cfebf3a 100755 --- a/openfl/interface/cli.py +++ b/openfl/interface/cli.py @@ -2,6 +2,7 @@ # Copyright (C) 2020-2023 Intel Corporation # SPDX-License-Identifier: Apache-2.0 """CLI module.""" + import logging import os import re diff --git a/openfl/interface/cli_helper.py b/openfl/interface/cli_helper.py index a326527158..065dd43da3 100644 --- a/openfl/interface/cli_helper.py +++ b/openfl/interface/cli_helper.py @@ -3,11 +3,11 @@ """Module with auxiliary CLI helper functions.""" + import os import re -import shutil from itertools import islice -from os import environ, stat +from os import environ from pathlib import Path from sys import argv @@ -35,25 +35,6 @@ def pretty(o): echo(style(f"{k:<{m}} : ", fg="blue") + style(f"{v}", fg="cyan")) -def tree(path): - """ - Print current directory file tree. - - Args: - path (str): The path of the directory. - """ - echo(f"+ {path}") - - for path in sorted(path.rglob("*")): - depth = len(path.relative_to(path).parts) - space = " " * depth - - if path.is_file(): - echo(f"{space}f {path.name}") - else: - echo(f"{space}d {path.name}") - - def print_tree( dir_path: Path, level: int = -1, @@ -108,101 +89,6 @@ def inner(dir_path: Path, prefix: str = "", level=-1): echo(f"\n{directories} directories" + (f", {files} files" if files else "")) -def copytree( - src, - dst, - symlinks=False, - ignore=None, - ignore_dangling_symlinks=False, - dirs_exist_ok=False, -): - """From Python 3.8 'shutil' which include 'dirs_exist_ok' option. - - Args: - src (str): The source directory. - dst (str): The destination directory. - symlinks (bool, optional): Whether to copy symlinks. Defaults to False. - ignore (callable, optional): A function that takes a directory name - and filenames as input parameters and returns a list of names to - ignore. Defaults to None. - ignore_dangling_symlinks (bool, optional): Whether to ignore dangling - symlinks. Defaults to False. - dirs_exist_ok (bool, optional): Whether to raise an exception in case - dst or any missing parent directory already exists. Defaults to - False. - """ - - with os.scandir(src) as itr: - entries = list(itr) - - copy_function = shutil.copy2 - - def _copytree(): - if ignore is not None: - ignored_names = ignore(os.fspath(src), [x.name for x in entries]) - else: - ignored_names = set() - - os.makedirs(dst, exist_ok=dirs_exist_ok) - errors = [] - use_srcentry = copy_function is shutil.copy2 or copy_function is shutil.copy - - for srcentry in entries: - if srcentry.name in ignored_names: - continue - srcname = os.path.join(src, srcentry.name) - dstname = os.path.join(dst, srcentry.name) - srcobj = srcentry if use_srcentry else srcname - try: - is_symlink = srcentry.is_symlink() - if is_symlink and os.name == "nt": - lstat = srcentry.stat(follow_symlinks=False) - if lstat.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT: - is_symlink = False - if is_symlink: - linkto = os.readlink(srcname) - if symlinks: - os.symlink(linkto, dstname) - shutil.copystat(srcobj, dstname, follow_symlinks=not symlinks) - else: - if not os.path.exists(linkto) and ignore_dangling_symlinks: - continue - if srcentry.is_dir(): - copytree( - srcobj, - dstname, - symlinks, - ignore, - dirs_exist_ok=dirs_exist_ok, - ) - else: - copy_function(srcobj, dstname) - elif srcentry.is_dir(): - copytree( - srcobj, - dstname, - symlinks, - ignore, - dirs_exist_ok=dirs_exist_ok, - ) - else: - copy_function(srcobj, dstname) - except OSError as why: - errors.append((srcname, dstname, str(why))) - except Exception as err: - errors.extend(err.args[0]) - try: - shutil.copystat(src, dst) - except OSError as why: - if getattr(why, "winerror", None) is None: - errors.append((src, dst, str(why))) - if errors: - raise Exception(errors) - return dst - - return _copytree() - - def get_workspace_parameter(name): """Get a parameter from the workspace config file (.workspace). diff --git a/openfl/interface/collaborator.py b/openfl/interface/collaborator.py index 9bad1e9716..862ae4db84 100644 --- a/openfl/interface/collaborator.py +++ b/openfl/interface/collaborator.py @@ -3,6 +3,7 @@ """Collaborator module.""" + import os import sys from glob import glob @@ -403,16 +404,13 @@ def certify(collaborator_name, silent, request_pkg=None, import_=False): signing_crt = read_crt(CERT_DIR / signing_crt_path) - echo( - "The CSR Hash for file " - + style(f"{file_name}.csr", fg="green") - + " = " - + style(f"{csr_hash}", fg="red") - ) + echo(f"The CSR Hash for file {file_name}.csr is {csr_hash}") if silent: - echo(" Signing COLLABORATOR certificate") - echo(" Warning: manual check of certificate hashes is bypassed in silent mode.") + echo( + "Signing COLLABORATOR certificate, " + "Warning: manual check of certificate hashes is bypassed in silent mode." + ) signed_col_cert = sign_certificate(csr, signing_key, signing_crt.subject) write_crt(signed_col_cert, f"{cert_name}.crt") register_collaborator(CERT_DIR / "client" / f"{file_name}.crt") @@ -458,13 +456,17 @@ def certify(collaborator_name, silent, request_pkg=None, import_=False): rmtree(tmp_dir) else: - # Copy the signed certificate and cert chain into PKI_DIR - previous_crts = glob(f"{CERT_DIR}/client/*.crt") - unpack_archive(import_, extract_dir=CERT_DIR) - updated_crts = glob(f"{CERT_DIR}/client/*.crt") - cert_difference = list(set(updated_crts) - set(previous_crts)) - if len(cert_difference) != 0: - crt = basename(cert_difference[0]) - echo(f"Certificate {crt} installed to PKI directory") - else: - echo("Certificate updated in the PKI directory") + _import_certificates(import_) + + +def _import_certificates(archive: str): + # Copy the signed certificate and cert chain into PKI_DIR + previous_crts = glob(f"{CERT_DIR}/client/*.crt") + unpack_archive(archive, extract_dir=CERT_DIR) + updated_crts = glob(f"{CERT_DIR}/client/*.crt") + cert_difference = list(set(updated_crts) - set(previous_crts)) + if len(cert_difference) != 0: + crt = basename(cert_difference[0]) + echo(f"Certificate {crt} installed to PKI directory") + else: + echo("Certificate updated in the PKI directory") diff --git a/openfl/interface/director.py b/openfl/interface/director.py index fb1460539e..bd73d865c1 100644 --- a/openfl/interface/director.py +++ b/openfl/interface/director.py @@ -3,6 +3,7 @@ """Director CLI.""" + import logging import shutil import sys diff --git a/openfl/interface/envoy.py b/openfl/interface/envoy.py index 1d99061f40..4e35391bba 100644 --- a/openfl/interface/envoy.py +++ b/openfl/interface/envoy.py @@ -3,6 +3,7 @@ """Envoy CLI.""" + import logging import shutil import sys diff --git a/openfl/interface/experimental.py b/openfl/interface/experimental.py index e2cf08aa78..67c77073fc 100644 --- a/openfl/interface/experimental.py +++ b/openfl/interface/experimental.py @@ -3,6 +3,7 @@ """Experimental CLI.""" + from logging import getLogger from pathlib import Path from subprocess import check_call diff --git a/openfl/interface/interactive_api/experiment.py b/openfl/interface/interactive_api/experiment.py index ce970abaaf..111c6f1238 100644 --- a/openfl/interface/interactive_api/experiment.py +++ b/openfl/interface/interactive_api/experiment.py @@ -3,6 +3,7 @@ """Python low-level API module.""" + import os import time from collections import defaultdict @@ -355,7 +356,7 @@ def start( else: self.logger.info("Experiment could not be submitted to the director.") - def define_task_assigner(self, task_keeper, rounds_to_train): + def define_task_assigner(self, task_keeper, rounds_to_train): # noqa: C901 """Define task assigner by registered tasks. This method defines a task assigner based on the registered tasks. @@ -408,6 +409,7 @@ def assigner(collaborators, round_number, **kwargs): return tasks_by_collaborator return assigner + elif not is_train_task_exist and self.is_validate_task_exist: def assigner(collaborators, round_number, **kwargs): @@ -419,6 +421,7 @@ def assigner(collaborators, round_number, **kwargs): return tasks_by_collaborator return assigner + elif is_train_task_exist and not self.is_validate_task_exist: raise Exception("You should define validate task!") else: @@ -568,9 +571,9 @@ def _prepare_plan( # Collaborator part self.plan.config["collaborator"]["settings"]["delta_updates"] = delta_updates self.plan.config["collaborator"]["settings"]["opt_treatment"] = opt_treatment - self.plan.config["collaborator"]["settings"][ - "device_assignment_policy" - ] = device_assignment_policy + self.plan.config["collaborator"]["settings"]["device_assignment_policy"] = ( + device_assignment_policy + ) # DataLoader part for setting, value in data_loader.kwargs.items(): diff --git a/openfl/interface/model.py b/openfl/interface/model.py index 9852124c6d..3703581899 100644 --- a/openfl/interface/model.py +++ b/openfl/interface/model.py @@ -3,6 +3,7 @@ """Model CLI module.""" + from logging import getLogger from pathlib import Path from typing import Union diff --git a/openfl/interface/plan.py b/openfl/interface/plan.py index 04f5dd9da8..93d09c02a5 100644 --- a/openfl/interface/plan.py +++ b/openfl/interface/plan.py @@ -3,6 +3,7 @@ """Plan module.""" + import os import sys from logging import getLogger diff --git a/openfl/interface/tutorial.py b/openfl/interface/tutorial.py index 2c1975f576..e3de9ee7b0 100644 --- a/openfl/interface/tutorial.py +++ b/openfl/interface/tutorial.py @@ -3,6 +3,7 @@ """Tutorial module.""" + from logging import getLogger from os import environ, sep from subprocess import check_call # nosec diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index b138ad67db..52129a967e 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -3,6 +3,7 @@ """Workspace module.""" + import logging import os import shutil @@ -15,9 +16,8 @@ from sys import executable from typing import Union -from click import Choice +from click import Choice, echo, group, option, pass_context from click import Path as ClickPath -from click import echo, group, option, pass_context from cryptography.hazmat.primitives import serialization from openfl.cryptography.ca import generate_root_cert, generate_signing_csr, sign_certificate @@ -395,9 +395,9 @@ def export_() -> str: type=str, required=False, help=( - "Path to an enclave signing key. If not provided, a key will be auto-generated in the workspace. " - "Note that this command builds a TEE-ready image, key is NOT packaged along with the image. " - "You have the flexibility to not run inside a TEE later." + "Path to an enclave signing key. If not provided, a key will be auto-generated in the " + "workspace. Note that this command builds a TEE-ready image, key is NOT packaged along " + "with the image. You have the flexibility to not run inside a TEE later." ), ) @option( @@ -510,7 +510,7 @@ def _execute(cmd: str, verbose=True) -> None: """ logging.info(f"Executing: {cmd}") process = subprocess.Popen(cmd, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) - stdout_log = list() + stdout_log = [] for line in process.stdout: msg = line.rstrip().decode("utf-8") stdout_log.append(msg) diff --git a/openfl/native/fastestimator.py b/openfl/native/fastestimator.py index b794d30387..a95db185d0 100644 --- a/openfl/native/fastestimator.py +++ b/openfl/native/fastestimator.py @@ -3,6 +3,7 @@ """FederatedFastEstimator module.""" + import os from logging import getLogger from pathlib import Path @@ -44,14 +45,12 @@ def __init__(self, estimator, override_config: dict = None, **kwargs): if override_config: fx.update_plan(override_config) - def fit(self): + def fit(self): # noqa: C901 """Runs the estimator in federated mode.""" - file = Path(__file__).resolve() # interface root, containing command modules root = file.parent.resolve() work = Path.cwd().resolve() - path.append(str(root)) path.insert(0, str(work)) diff --git a/openfl/native/native.py b/openfl/native/native.py index c266c73ac8..117058abbb 100644 --- a/openfl/native/native.py +++ b/openfl/native/native.py @@ -7,6 +7,7 @@ This file defines openfl entrypoints to be used directly through python (not CLI) """ + import json import logging import os @@ -85,7 +86,7 @@ def flatten(config, return_complete=False): return flattened_config -def update_plan(override_config, plan=None, resolve=True): +def update_plan(override_config, plan=None, resolve=True): # noqa: C901 """Updates the plan with the provided override and saves it to disk. For a list of available override options, call `fx.get_plan()` diff --git a/openfl/pipelines/eden_pipeline.py b/openfl/pipelines/eden_pipeline.py index 61c51cd8d9..fe05bd3af5 100644 --- a/openfl/pipelines/eden_pipeline.py +++ b/openfl/pipelines/eden_pipeline.py @@ -438,7 +438,7 @@ def rand_diag(self, size, seed): res = torch.zeros(size_scaled * bools_in_float32, device=self.device) s = 0 - for i in range(bools_in_float32): + for _ in range(bools_in_float32): res[s : s + size_scaled] = r & mask s += size_scaled r >>= shift diff --git a/openfl/pipelines/stc_pipeline.py b/openfl/pipelines/stc_pipeline.py index 61e2fcf704..4a5a9126ce 100644 --- a/openfl/pipelines/stc_pipeline.py +++ b/openfl/pipelines/stc_pipeline.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """STCPipelinemodule.""" + import copy import gzip as gz diff --git a/openfl/plugins/frameworks_adapters/flax_adapter.py b/openfl/plugins/frameworks_adapters/flax_adapter.py index 1e6fd1e203..ce9399d7e8 100644 --- a/openfl/plugins/frameworks_adapters/flax_adapter.py +++ b/openfl/plugins/frameworks_adapters/flax_adapter.py @@ -3,6 +3,7 @@ """Custom model DeviceArray - JAX Numpy adapter.""" + import jax import jax.numpy as jnp import numpy as np @@ -131,7 +132,7 @@ def _update_weights(state_dict, tensor_dict, prefix, suffix=None): """ dict_prefix = f"{prefix}_{suffix}" if suffix is not None else f"{prefix}" for layer_name, param_obj in state_dict.items(): - for param_name, value in param_obj.items(): + for param_name, _ in param_obj.items(): key = "*".join([dict_prefix, layer_name, param_name]) if key in tensor_dict: state_dict[layer_name][param_name] = tensor_dict[key] diff --git a/openfl/plugins/frameworks_adapters/keras_adapter.py b/openfl/plugins/frameworks_adapters/keras_adapter.py index 971be7840e..a484372ad0 100644 --- a/openfl/plugins/frameworks_adapters/keras_adapter.py +++ b/openfl/plugins/frameworks_adapters/keras_adapter.py @@ -3,6 +3,7 @@ """Keras Framework Adapter plugin.""" + from logging import getLogger from packaging import version diff --git a/openfl/plugins/frameworks_adapters/pytorch_adapter.py b/openfl/plugins/frameworks_adapters/pytorch_adapter.py index a4b33fd73f..60a6db54f4 100644 --- a/openfl/plugins/frameworks_adapters/pytorch_adapter.py +++ b/openfl/plugins/frameworks_adapters/pytorch_adapter.py @@ -3,6 +3,7 @@ """Pytorch Framework Adapter plugin.""" + from copy import deepcopy import numpy as np diff --git a/openfl/protocols/interceptors.py b/openfl/protocols/interceptors.py index 3549529da6..e6a9b57b3f 100644 --- a/openfl/protocols/interceptors.py +++ b/openfl/protocols/interceptors.py @@ -3,6 +3,7 @@ """gRPC interceptors module.""" + import collections import grpc diff --git a/openfl/transport/grpc/aggregator_client.py b/openfl/transport/grpc/aggregator_client.py index 6713107c2b..4b61cb888b 100644 --- a/openfl/transport/grpc/aggregator_client.py +++ b/openfl/transport/grpc/aggregator_client.py @@ -173,9 +173,12 @@ class AggregatorGRPCClient: use_tls (bool): Whether to use TLS for the connection. require_client_auth (bool): Whether to enable client-side authentication, i.e. mTLS. Ignored if `use_tls=False`. - root_certificate (str): The path to the root certificate for the TLS connection, ignored if `use_tls=False`. - certificate (str): The path to the client's certificate for the TLS connection, ignored if `use_tls=False`. - private_key (str): The path to the client's private key for the TLS connection, ignored if `use_tls=False`. + root_certificate (str): The path to the root certificate for the TLS connection, ignored if + `use_tls=False`. + certificate (str): The path to the client's certificate for the TLS connection, ignored if + `use_tls=False`. + private_key (str): The path to the client's private key for the TLS connection, ignored if + `use_tls=False`. aggregator_uuid (str): The UUID of the aggregator. federation_uuid (str): The UUID of the federation. single_col_cert_common_name (str): The common name on the diff --git a/openfl/transport/grpc/aggregator_server.py b/openfl/transport/grpc/aggregator_server.py index 19d156338f..bfae10351b 100644 --- a/openfl/transport/grpc/aggregator_server.py +++ b/openfl/transport/grpc/aggregator_server.py @@ -31,9 +31,12 @@ class AggregatorGRPCServer(aggregator_pb2_grpc.AggregatorServicer): use_tls (bool): Whether to use TLS for the connection. require_client_auth (bool): Whether to enable client-side authentication, i.e. mTLS. Ignored if `use_tls=False`. - root_certificate (str): The path to the root certificate for the TLS connection, ignored if `use_tls=False`. - certificate (str): The path to the client's certificate for the TLS connection, ignored if `use_tls=False`. - private_key (str): The path to the client's private key for the TLS connection, ignored if `use_tls=False`. + root_certificate (str): The path to the root certificate for the TLS connection, ignored if + `use_tls=False`. + certificate (str): The path to the client's certificate for the TLS connection, ignored if + `use_tls=False`. + private_key (str): The path to the client's private key for the TLS connection, ignored if + `use_tls=False`. server (grpc.Server): The gRPC server. server_credentials (grpc.ServerCredentials): The server's credentials. """ diff --git a/openfl/utilities/ca.py b/openfl/utilities/ca.py index ddedef1016..d522d541cd 100644 --- a/openfl/utilities/ca.py +++ b/openfl/utilities/ca.py @@ -3,6 +3,7 @@ """Generic check functions.""" + import os diff --git a/openfl/utilities/ca/ca.py b/openfl/utilities/ca/ca.py index acf3a50e01..e75d68c491 100644 --- a/openfl/utilities/ca/ca.py +++ b/openfl/utilities/ca/ca.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """CA module.""" + import base64 import json import os diff --git a/openfl/utilities/data_splitters/__init__.py b/openfl/utilities/data_splitters/__init__.py index 5322f46188..fdf28bb45d 100644 --- a/openfl/utilities/data_splitters/__init__.py +++ b/openfl/utilities/data_splitters/__init__.py @@ -3,6 +3,7 @@ """openfl.utilities.data package.""" + from openfl.utilities.data_splitters.data_splitter import DataSplitter from openfl.utilities.data_splitters.numpy import ( DirichletNumPyDataSplitter, diff --git a/openfl/utilities/data_splitters/data_splitter.py b/openfl/utilities/data_splitters/data_splitter.py index 19333be59a..216607eb3d 100644 --- a/openfl/utilities/data_splitters/data_splitter.py +++ b/openfl/utilities/data_splitters/data_splitter.py @@ -3,6 +3,7 @@ """openfl.utilities.data_splitters.data_splitter module.""" + from abc import ABC, abstractmethod from typing import Iterable, List, TypeVar diff --git a/openfl/utilities/fed_timer.py b/openfl/utilities/fed_timer.py index a8ecb9e45f..3d8770ec24 100644 --- a/openfl/utilities/fed_timer.py +++ b/openfl/utilities/fed_timer.py @@ -29,7 +29,7 @@ class CustomThread(Thread): kwargs (dict): The keyword arguments to pass to the target function. """ - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): + def __init__(self, group=None, target=None, name=None, args=None, kwargs=None): """Initialize a CustomThread object. Args: diff --git a/openfl/utilities/optimizers/keras/fedprox.py b/openfl/utilities/optimizers/keras/fedprox.py index ef1634d77c..2e87648c20 100644 --- a/openfl/utilities/optimizers/keras/fedprox.py +++ b/openfl/utilities/optimizers/keras/fedprox.py @@ -3,6 +3,7 @@ """FedProx Keras optimizer module.""" + import tensorflow as tf import tensorflow.keras as keras from tensorflow.python.ops import standard_ops diff --git a/openfl/utilities/optimizers/numpy/base_optimizer.py b/openfl/utilities/optimizers/numpy/base_optimizer.py index d748727dea..bbbae93ce4 100644 --- a/openfl/utilities/optimizers/numpy/base_optimizer.py +++ b/openfl/utilities/optimizers/numpy/base_optimizer.py @@ -3,6 +3,7 @@ """Base abstract optimizer class module.""" + import abc from importlib import import_module from os.path import splitext diff --git a/openfl/utilities/optimizers/torch/__init__.py b/openfl/utilities/optimizers/torch/__init__.py index 77e989cdd4..7b595b3d7a 100644 --- a/openfl/utilities/optimizers/torch/__init__.py +++ b/openfl/utilities/optimizers/torch/__init__.py @@ -3,6 +3,7 @@ """PyTorch optimizers package.""" + from importlib import util if util.find_spec("torch") is not None: diff --git a/openfl/utilities/workspace.py b/openfl/utilities/workspace.py index e88f431625..e19b03cf16 100644 --- a/openfl/utilities/workspace.py +++ b/openfl/utilities/workspace.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Workspace utils module.""" + import logging import os import shutil diff --git a/pyproject.toml b/pyproject.toml index b38d4235fd..68a42f699b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,27 @@ -[tool.black] +[tool.ruff] line-length = 100 -[tool.isort] -profile = "black" -force_single_line = "False" -line_length = 100 \ No newline at end of file +[tool.ruff.lint] +select = [ + "B", # Bandit security checks (e.g., detecting insecure function use). + "C", # Cyclomatic complexity, used to flag overly complex functions. + "E", # PEP8 errors (e.g., style issues). + "F", # Pyflakes errors, like unused imports or undefined names. + "W", # PEP8 warnings (e.g., deprecations). + "B9", # Bugbear, for additional warnings about potentially error-prone code. + "I", # isort +] +ignore = [ + "E266", # too many leading '#' for block comments + "E741", # ambiguous variable name + "E731", # do not assign a `lambda` expression, use a `def` + "B904", # ignore exception distinguishing + "B006", # mutable data structures as default args +] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["E501", "F401"] # lines too long; imported but unused + +[tool.ruff.lint.isort] +force-single-line = false +known-first-party = ["openfl"] \ No newline at end of file diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5874a2e513..0000000000 --- a/setup.cfg +++ /dev/null @@ -1,39 +0,0 @@ -[flake8] -ignore = - # Conflicts with black - E203 - # Line break occurred before a binary operator. Update by W504 Line - W503 - # Allow "import torch.nn.functional as F" - N812 - # Line length, handled separately by max-line-length - E501 - # too many leading '#' for block comments - E266 - - -per-file-ignores = - # Unused imports in __init__.py are OK - **/__init__.py:F401 - -exclude = - *_pb2*, - .git, - __pycache__, - build, - dist, - .venv - -max-line-length = 100 - -copyright-check = True - -# Enable specific checks or plugins -# B: Bandit security checks (e.g., detecting insecure function use). -# C: Cyclomatic complexity, used to flag overly complex functions. -# E: PEP8 errors (e.g., style issues). -# F: Pyflakes errors, like unused imports or undefined names. -# W: PEP8 warnings (e.g., stylistic issues). -# T4: Type checking from third-party tools (like mypy). -# B9: Bugbear, for additional warnings about potentially error-prone code. -select = B,C,E,F,W,T4,B9 diff --git a/shell/format.sh b/shell/format.sh index 5d8f2c1547..0033b0d52c 100755 --- a/shell/format.sh +++ b/shell/format.sh @@ -6,8 +6,6 @@ base_dir=$(dirname $(dirname $0)) # Run the pre-commit checks pre-commit run --all-files -isort --sp "${base_dir}/pyproject.toml" openfl +ruff check --config "${base_dir}/pyproject.toml" --fix openfl/ -black --config "${base_dir}/pyproject.toml" openfl - -flake8 --config "${base_dir}/setup.cfg" openfl \ No newline at end of file +ruff format --config "${base_dir}/pyproject.toml" openfl/ \ No newline at end of file diff --git a/shell/lint.sh b/shell/lint.sh index f3cee9c4a1..ac0b97e0a5 100755 --- a/shell/lint.sh +++ b/shell/lint.sh @@ -6,8 +6,6 @@ base_dir=$(dirname $(dirname $0)) # Run the pre-commit checks pre-commit run --all-files -isort --sp "${base_dir}/pyproject.toml" --check openfl +ruff check --config "${base_dir}/pyproject.toml" openfl/ -black --config "${base_dir}/pyproject.toml" --check openfl - -flake8 --config "${base_dir}/setup.cfg" --show-source openfl \ No newline at end of file +ruff format --check --config "${base_dir}/pyproject.toml" openfl/ \ No newline at end of file