diff --git a/docs/source/conf.py b/docs/source/conf.py index 33e4e6fbd..ceb66dcaf 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -191,9 +191,7 @@ def setup(app): # In Sphinx 1.8 it was renamed to `add_css_file`, 1.7 and prior it is # `add_stylesheet` (deprecated in 1.8). - add_css = getattr( - app, "add_css_file", getattr(app, "add_stylesheet", None) - ) # noqa B009 + add_css = getattr(app, "add_css_file", getattr(app, "add_stylesheet", None)) # noqa B009 for css_file in html_css_files: add_css(css_file) diff --git a/scripts/collect_env.py b/scripts/collect_env.py index 7890d8f65..ae6f1f4ae 100644 --- a/scripts/collect_env.py +++ b/scripts/collect_env.py @@ -33,8 +33,9 @@ def run( args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True ) raw_output, raw_err = p.communicate() - raw_output, raw_err = raw_output.strip().decode("utf-8"), raw_err.strip().decode( - "utf-8" + raw_output, raw_err = ( + raw_output.strip().decode("utf-8"), + raw_err.strip().decode("utf-8"), ) rc = p.returncode if rc != 0: diff --git a/scripts/component_integration_tests.py b/scripts/component_integration_tests.py index 2eb3013d4..b50f00790 100755 --- a/scripts/component_integration_tests.py +++ b/scripts/component_integration_tests.py @@ -10,6 +10,7 @@ """ Kubernetes integration tests. """ + import argparse import logging import os diff --git a/tools/linter/adapters/pyre_linter.py b/tools/linter/adapters/pyre_linter.py index d5c9ad840..99f2c3b8d 100644 --- a/tools/linter/adapters/pyre_linter.py +++ b/tools/linter/adapters/pyre_linter.py @@ -109,7 +109,9 @@ def main() -> None: level=( logging.NOTSET if args.verbose - else logging.DEBUG if len(args.filenames) < 1000 else logging.INFO + else logging.DEBUG + if len(args.filenames) < 1000 + else logging.INFO ), stream=sys.stderr, ) diff --git a/tools/linter/adapters/ufmt_linter.py b/tools/linter/adapters/ufmt_linter.py index 936f4f98b..1f6ad8cf2 100644 --- a/tools/linter/adapters/ufmt_linter.py +++ b/tools/linter/adapters/ufmt_linter.py @@ -119,7 +119,9 @@ def main() -> None: level=( logging.NOTSET if args.verbose - else logging.DEBUG if len(args.filenames) < 1000 else logging.INFO + else logging.DEBUG + if len(args.filenames) < 1000 + else logging.INFO ), stream=sys.stderr, ) diff --git a/torchx/cli/test/cmd_run_test.py b/torchx/cli/test/cmd_run_test.py index dbbecca66..b55f17c8f 100644 --- a/torchx/cli/test/cmd_run_test.py +++ b/torchx/cli/test/cmd_run_test.py @@ -51,14 +51,12 @@ def tearDown(self) -> None: torchxconfig.called_args = set() def test_run_with_multiple_scheduler_args(self) -> None: - args = ["--scheduler_args", "first_args", "--scheduler_args", "second_args"] with self.assertRaises(SystemExit) as cm: self.parser.parse_args(args) self.assertEqual(cm.exception.code, 1) def test_run_with_multiple_schedule_args(self) -> None: - args = [ "--scheduler", "local_cwd", diff --git a/torchx/components/dist.py b/torchx/components/dist.py index 817534a25..feee5f9a9 100644 --- a/torchx/components/dist.py +++ b/torchx/components/dist.py @@ -56,6 +56,7 @@ Components APIs ----------------- """ + import os import re import shlex diff --git a/torchx/components/structured_arg.py b/torchx/components/structured_arg.py index 29a26ea1b..f0d440962 100644 --- a/torchx/components/structured_arg.py +++ b/torchx/components/structured_arg.py @@ -25,6 +25,7 @@ with an experiment tracker. The ``/`` delimiter is a natural way to group runs within experiments. """ + import warnings from dataclasses import dataclass from pathlib import Path diff --git a/torchx/runner/config.py b/torchx/runner/config.py index a7c0f4e8c..842cc01d6 100644 --- a/torchx/runner/config.py +++ b/torchx/runner/config.py @@ -157,6 +157,7 @@ def my_component(a: int) -> specs.AppDef: you want to keep personal config overrides on top of a project defined default. """ + import configparser as configparser import logging import os diff --git a/torchx/schedulers/aws_batch_scheduler.py b/torchx/schedulers/aws_batch_scheduler.py index a354c57bc..46449c83f 100644 --- a/torchx/schedulers/aws_batch_scheduler.py +++ b/torchx/schedulers/aws_batch_scheduler.py @@ -36,6 +36,7 @@ https://docs.aws.amazon.com/AmazonECR/latest/userguide/getting-started-cli.html#cli-create-repository for how to create a image repository. """ + import getpass import re import threading @@ -155,7 +156,7 @@ def resource_requirements_from_resource(resource: Resource) -> List[Dict[str, st def resource_from_resource_requirements( - resource_requirements: List[Dict[str, str]] + resource_requirements: List[Dict[str, str]], ) -> Resource: resrc_req = { ResourceType.from_str(r["type"]): int(r["value"]) for r in resource_requirements diff --git a/torchx/schedulers/kubernetes_mcad_scheduler.py b/torchx/schedulers/kubernetes_mcad_scheduler.py index 2ac31952e..e3bc71c42 100644 --- a/torchx/schedulers/kubernetes_mcad_scheduler.py +++ b/torchx/schedulers/kubernetes_mcad_scheduler.py @@ -17,8 +17,8 @@ TorchX Kubernetes_MCAD scheduler depends on AppWrapper + MCAD. -Install MCAD: -See deploying Multi-Cluster-Application-Dispatcher guide +Install MCAD: +See deploying Multi-Cluster-Application-Dispatcher guide https://github.com/project-codeflare/multi-cluster-app-dispatcher/blob/main/doc/deploy/deployment.md This implementation requires MCAD v1.34.1 or higher. @@ -478,8 +478,10 @@ def get_unique_truncated_appid(app: AppDef) -> str: unique_id_size = default_size if size > default_size else size if unique_id_size <= 3: - msg = "Name size has too many characters for some Kubernetes objects. Truncating \ + msg = ( + "Name size has too many characters for some Kubernetes objects. Truncating \ application name." + ) warnings.warn(msg) end = 63 - uid_chars - pg_chars substring = app.name[0:end] diff --git a/torchx/schedulers/lsf_scheduler.py b/torchx/schedulers/lsf_scheduler.py index 102a8f417..861a0f680 100644 --- a/torchx/schedulers/lsf_scheduler.py +++ b/torchx/schedulers/lsf_scheduler.py @@ -23,6 +23,7 @@ See the LSF documentation for more details: https://www.ibm.com/docs/en/cloud-private/3.2.x?topic=paks-spectrum-lsf-community-edition """ + import os.path import re import subprocess diff --git a/torchx/schedulers/ray/ray_driver.py b/torchx/schedulers/ray/ray_driver.py index 8557cdf29..ce6717f31 100644 --- a/torchx/schedulers/ray/ray_driver.py +++ b/torchx/schedulers/ray/ray_driver.py @@ -18,6 +18,7 @@ _step function, this give more flexibility to us if we want to bette handle the node failures. """ + import json import logging import os @@ -148,12 +149,12 @@ def __init__(self, replicas: List[RayActor]) -> None: else: self.min_replicas = replicas[0].min_replicas # pyre-ignore[8] - self.placement_groups: List[PlacementGroup] = ( - [] - ) # all the placement groups, shall never change - self.actor_info_of_id: Dict[str, ActorInfo] = ( - {} - ) # store the info used to recover an actor + self.placement_groups: List[ + PlacementGroup + ] = [] # all the placement groups, shall never change + self.actor_info_of_id: Dict[ + str, ActorInfo + ] = {} # store the info used to recover an actor self.active_tasks: List["ray.ObjectRef"] = [] # list of active tasks self.terminating: bool = False # if the job has finished and being terminated diff --git a/torchx/schedulers/slurm_scheduler.py b/torchx/schedulers/slurm_scheduler.py index 331fd8611..f7563e515 100644 --- a/torchx/schedulers/slurm_scheduler.py +++ b/torchx/schedulers/slurm_scheduler.py @@ -11,6 +11,7 @@ This contains the TorchX Slurm scheduler which can be used to run TorchX components on a Slurm cluster. """ + import csv import json import logging diff --git a/torchx/schedulers/test/kubernetes_scheduler_test.py b/torchx/schedulers/test/kubernetes_scheduler_test.py index a03b2834b..e871ff96c 100644 --- a/torchx/schedulers/test/kubernetes_scheduler_test.py +++ b/torchx/schedulers/test/kubernetes_scheduler_test.py @@ -112,9 +112,7 @@ def test_app_to_resource_resolved_macros(self) -> None: resource = app_to_resource(app, "test_queue", service_account=None) actual_cmd = ( # pyre-ignore [16] - resource["spec"]["tasks"][0]["template"] - .spec.containers[0] - .command + resource["spec"]["tasks"][0]["template"].spec.containers[0].command ) expected_cmd = [ "main", diff --git a/torchx/schedulers/test/ray_scheduler_test.py b/torchx/schedulers/test/ray_scheduler_test.py index 1e7d181b5..66d966b55 100644 --- a/torchx/schedulers/test/ray_scheduler_test.py +++ b/torchx/schedulers/test/ray_scheduler_test.py @@ -584,9 +584,7 @@ def test_ray_driver_elasticity(self) -> None: self.assertIsNotNone(driver.rank_0_port) # 3-2 - terminal = ( - driver._step() - ) # actor 1 finished, actor 2 has been scheduled yet, usually, the driver stops here + terminal = driver._step() # actor 1 finished, actor 2 has been scheduled yet, usually, the driver stops here self.assertEqual(terminal, True) self.assertEqual(driver.command_actors_count, 0) self.assertEqual(len(driver.active_tasks), 1) # actor schedule task diff --git a/torchx/specs/__init__.py b/torchx/specs/__init__.py index c43cfa0c9..2895fd099 100644 --- a/torchx/specs/__init__.py +++ b/torchx/specs/__init__.py @@ -12,6 +12,7 @@ used by components to define the apps which can then be launched via a TorchX scheduler or pipeline adapter. """ + import difflib from typing import Callable, Dict, Optional diff --git a/torchx/specs/named_resources_generic.py b/torchx/specs/named_resources_generic.py index 2d93ecf75..792df3041 100644 --- a/torchx/specs/named_resources_generic.py +++ b/torchx/specs/named_resources_generic.py @@ -37,6 +37,7 @@ have 8 cpus. """ + from typing import Callable, Mapping from torchx.specs.api import Resource diff --git a/torchx/specs/test/api_test.py b/torchx/specs/test/api_test.py index 389102547..6cfaeb301 100644 --- a/torchx/specs/test/api_test.py +++ b/torchx/specs/test/api_test.py @@ -301,7 +301,6 @@ async def update(value: str, time_seconds: int) -> str: self.assertEqual("nentry", default.entrypoint) def test_concurrent_override_role(self) -> None: - def delay(value: Tuple[str, str], time_seconds: int) -> Tuple[str, str]: time.sleep(time_seconds) return value diff --git a/torchx/test/fixtures.py b/torchx/test/fixtures.py index 58e6a1d3e..9f9c6555c 100644 --- a/torchx/test/fixtures.py +++ b/torchx/test/fixtures.py @@ -9,6 +9,7 @@ """ Useful test fixtures (classes that you can subclass your python ``unittest.TestCase``) """ + import os import shutil import sys diff --git a/torchx/tracker/api.py b/torchx/tracker/api.py index 28f8d02c3..927a0e2f2 100644 --- a/torchx/tracker/api.py +++ b/torchx/tracker/api.py @@ -179,7 +179,7 @@ def _extract_tracker_name_and_config_from_environ() -> Mapping[str, Optional[str def build_trackers( - factory_and_config: Mapping[str, Optional[str]] + factory_and_config: Mapping[str, Optional[str]], ) -> Iterable[TrackerBase]: trackers = [] diff --git a/torchx/tracker/backend/test/fsspec_test.py b/torchx/tracker/backend/test/fsspec_test.py index 10ed27f5d..b716791c8 100644 --- a/torchx/tracker/backend/test/fsspec_test.py +++ b/torchx/tracker/backend/test/fsspec_test.py @@ -171,5 +171,7 @@ def test_create(self) -> None: tracker = create(f"file://{str(configfile)}") self.assertEqual( - tracker._path_builder.root_dir, tracker_root_path # pyre-ignore + # pyre-fixme[16]: `TrackerBase` has no attribute `_path_builder`. + tracker._path_builder.root_dir, + tracker_root_path, ) diff --git a/torchx/tracker/mlflow.py b/torchx/tracker/mlflow.py index c9f908640..bb3772fec 100644 --- a/torchx/tracker/mlflow.py +++ b/torchx/tracker/mlflow.py @@ -254,7 +254,10 @@ def run_ids(self, **kwargs: str) -> Iterable[str]: return [r.info.run_name for r in runs] def log_params_flat( - self, run_name: str, cfg: Any, key: str = "" # pyre-ignore[2] + self, + run_name: str, + cfg: Any, + key: str = "", # pyre-ignore[2] ) -> None: """ Designed to be primarily used with hydra-style config objects (e.g. dataclasses),