format codes with ruff rules (#321)

format codes with ruff rules --------- Signed-off-by: Jinzhe Zeng <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
deepmodeling · Apr 11, 2023 · f1b710f · f1b710f
1 parent 8e1fc1f
commit f1b710f
Show file tree

Hide file tree

Showing 74 changed files with 370 additions and 441 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -20,11 +20,12 @@ repos:
     rev: 23.3.0
     hooks:
     -   id: black-jupyter
--   repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+-   repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.0.260
     hooks:
-    - id: isort
-    files: \.py$
+    - id: ruff
+      args: ["--fix"]
 # numpydoc
 -   repo: https://github.com/Carreau/velin
     rev: 0.0.12

diff --git a/doc/conf.py b/doc/conf.py
@@ -11,7 +11,6 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import os
-import subprocess
 import sys
 from datetime import date
 

diff --git a/dpdispatcher/__init__.py b/dpdispatcher/__init__.py
@@ -13,7 +13,7 @@
     )
 except PermissionError:
     warnings.warn(
-        f"dpdispatcher.log meet permission error. redirect the log to ~/dpdispatcher.log"
+        "dpdispatcher.log meet permission error. redirect the log to ~/dpdispatcher.log"
     )
     dlogf = logging.FileHandler(
         os.path.join(os.path.expanduser("~"), "dpdispatcher.log")
@@ -56,10 +56,7 @@
 
 
 def info():
-    """
-    Show basic information about dpdispatcher, its location and version.
-    """
-
+    """Show basic information about dpdispatcher, its location and version."""
     print("DeepModeling\n------------")
     print("Version: " + __version__)
     print("Path:    " + ROOT_PATH)
@@ -73,3 +70,28 @@ def info():
         except ImportError:
             print("%10s %10s Not Found" % (modui, ""))
     print()
+
+
+__all__ = [
+    "__version__",
+    "DistributedShell",
+    "DpCloudServer",
+    "DpCloudServerContext",
+    "HDFSContext",
+    "LazyLocalContext",
+    "LocalContext",
+    "LSF",
+    "Machine",
+    "PBS",
+    "Shell",
+    "Slurm",
+    "SSHContext",
+    "Submission",
+    "Task",
+    "Torque",
+    "info",
+    "Lebesgue",
+    "LebesgueContext",
+    "Job",
+    "Resources",
+]
diff --git a/dpdispatcher/base_context.py b/dpdispatcher/base_context.py
@@ -1,5 +1,5 @@
 from abc import ABCMeta, abstractmethod
-from typing import List, Optional, Tuple
+from typing import List, Tuple
 
 from dargs import Argument
 

diff --git a/dpdispatcher/distributed_shell.py b/dpdispatcher/distributed_shell.py
@@ -1,5 +1,3 @@
-import subprocess as sp
-
 from dpdispatcher import dlog
 from dpdispatcher.JobStatus import JobStatus
 from dpdispatcher.machine import Machine
@@ -118,7 +116,7 @@ def gen_script_header(self, job):
         return shell_script_header
 
     def do_submit(self, job):
-        """submit th job to yarn using distributed shell
+        """Submit th job to yarn using distributed shell.
 
         Parameters
         ----------
@@ -130,7 +128,6 @@ def do_submit(self, job):
         job_id: string
             submit process id
         """
-
         script_str = self.gen_script(job)
         script_file_name = job.script_file_name
         job_id_name = job.job_hash + "_job_id"
@@ -161,7 +158,7 @@ def do_submit(self, job):
             )
         )
 
-        cmd = "{ nohup %s 1>%s 2>%s & } && echo $!" % (
+        cmd = "{{ nohup {} 1>{} 2>{} & }} && echo $!".format(
             submit_command,
             output_name,
             output_name,

diff --git a/dpdispatcher/dp_cloud_server_context.py b/dpdispatcher/dp_cloud_server_context.py
@@ -1,9 +1,7 @@
 #!/usr/bin/env python
-# coding: utf-8
 # %%
 import os
 import shutil
-import time
 import uuid
 from typing import List
 
@@ -18,7 +16,6 @@
 from .dpcloudserver import Client, zip_file
 
 # from zip_file import zip_files
-from .dpcloudserver.config import ALI_OSS_BUCKET_URL
 
 DP_CLOUD_SERVER_HOME_DIR = os.path.join(
     os.path.expanduser("~"), ".dpdispatcher/", "dp_cloud_server/"
@@ -247,7 +244,7 @@ def write_home_file(self, fname, write_str):
         return True
 
     def read_home_file(self, fname):
-        with open(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname), "r") as fp:
+        with open(os.path.join(DP_CLOUD_SERVER_HOME_DIR, fname)) as fp:
             ret = fp.read()
         return ret
 

diff --git a/dpdispatcher/dpcloudserver/__init__.py b/dpdispatcher/dpcloudserver/__init__.py
@@ -1 +1,3 @@
 from .client import Client
+
+__all__ = ["Client"]
diff --git a/dpdispatcher/dpcloudserver/client.py b/dpdispatcher/dpcloudserver/client.py
@@ -1,4 +1,3 @@
-import json
 import os
 import re
 import time
@@ -285,7 +284,7 @@ def get_tasks_list(self, group_id, per_page=30):
         page = 0
         while True:
             ret = self.get(
-                f"/brm/v1/job/list",
+                "/brm/v1/job/list",
                 params={
                     "groupId": group_id,
                     "page": page,

diff --git a/dpdispatcher/dpcloudserver/temp_test.py b/dpdispatcher/dpcloudserver/temp_test.py
@@ -53,7 +53,9 @@ def setUp(self):
     def test_commit_job(self):
         print("----------", sys._getframe().f_code.co_name)
         file_uuid = uuid.uuid1().hex
-        oss_task_zip = os.path.join("%s/%s/%s.zip" % ("indicate", file_uuid, file_uuid))
+        oss_task_zip = os.path.join(
+            "{}/{}/{}.zip".format("indicate", file_uuid, file_uuid)
+        )
         zip_path = "/home/felix/workplace/22_dpdispatcher/dpdispatcher-yfb/dpdispatcher/dpcloudserver/t.txt"
         zip_task_file = zip_path + ".zip"
         zip_files(zip_path, zip_task_file, [])

diff --git a/dpdispatcher/dpdisp.py b/dpdispatcher/dpdisp.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 
 
 def main():

diff --git a/dpdispatcher/hdfs_cli.py b/dpdispatcher/hdfs_cli.py
@@ -1,22 +1,20 @@
 # /usr/bin/python
-# -*- encoding=utf-8 -*-
 
 import os
-import sys
 
 from dpdispatcher.utils import run_cmd_with_all_output
 
 
-class HDFS(object):
-    """Fundamental class for HDFS basic manipulation"""
+class HDFS:
+    """Fundamental class for HDFS basic manipulation."""
 
     @staticmethod
     def exists(uri):
         """Check existence of hdfs uri
         Returns: True on exists
-        Raises: RuntimeError
+        Raises: RuntimeError.
         """
-        cmd = "hadoop fs -test -e {uri}".format(uri=uri)
+        cmd = f"hadoop fs -test -e {uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -40,10 +38,9 @@ def exists(uri):
     def remove(uri):
         """Check existence of hdfs uri
         Returns: True on exists
-        Raises: RuntimeError
+        Raises: RuntimeError.
         """
-
-        cmd = "hadoop fs -rm -r {uri}".format(uri=uri)
+        cmd = f"hadoop fs -rm -r {uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -64,9 +61,9 @@ def remove(uri):
     def mkdir(uri):
         """Make new hdfs directory
         Returns: True on success
-        Raises: RuntimeError
+        Raises: RuntimeError.
         """
-        cmd = "hadoop fs -mkdir -p {uri}".format(uri=uri)
+        cmd = f"hadoop fs -mkdir -p {uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -85,9 +82,8 @@ def mkdir(uri):
 
     @staticmethod
     def copy_from_local(local_path, to_uri):
-        """
-        Returns: True on success
-        Raises: on unexpected error
+        """Returns: True on success
+        Raises: on unexpected error.
         """
         # Make sure local_path is accessible
         if not os.path.exists(local_path) or not os.access(local_path, os.R_OK):
@@ -146,7 +142,7 @@ def copy_to_local(from_uri, local_path):
 
     @staticmethod
     def read_hdfs_file(uri):
-        cmd = "hadoop fs -text {uri}".format(uri=uri)
+        cmd = f"hadoop fs -text {uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:
@@ -165,7 +161,7 @@ def read_hdfs_file(uri):
 
     @staticmethod
     def move(from_uri, to_uri):
-        cmd = "hadoop fs -mv {furi} {turi}".format(furi=from_uri, turi=to_uri)
+        cmd = f"hadoop fs -mv {from_uri} {to_uri}"
         try:
             ret, out, err = run_cmd_with_all_output(cmd)
             if ret == 0:

diff --git a/dpdispatcher/hdfs_context.py b/dpdispatcher/hdfs_context.py
@@ -1,4 +1,3 @@
-import hashlib
 import os
 import shutil
 import tarfile
@@ -69,18 +68,19 @@ def _put_files(self, files, dereference=True):
         os.remove(from_f)
 
     def upload(self, submission, dereference=True):
-        """upload forward files and forward command files to HDFS root dir
+        """Upload forward files and forward command files to HDFS root dir.
 
         Parameters
         ----------
         submission : Submission class instance
             represents a collection of tasks, such as forward file names
+        dereference : bool
+            whether to dereference symbolic links
 
         Returns
         -------
         none
         """
-
         file_list = []
 
         for task in submission.belonging_tasks:
@@ -113,26 +113,31 @@ def upload(self, submission, dereference=True):
     def download(
         self, submission, check_exists=False, mark_failure=True, back_error=False
     ):
-        """download backward files from HDFS root dir
+        """Download backward files from HDFS root dir.
 
         Parameters
         ----------
         submission : Submission class instance
             represents a collection of tasks, such as backward file names
+        check_exists : bool
+            whether to check if the file exists
+        mark_failure : bool
+            whether to mark the task as failed if the file does not exist
+        back_error : bool
+            whether to download error files
 
         Returns
         -------
         none
         """
-
         cwd = os.getcwd()
 
         # download all hdfs files to tmp dir
         gz_dir = os.path.join(self.local_root, "tmp")
         if os.path.exists(gz_dir):
             shutil.rmtree(gz_dir, ignore_errors=True)
         os.mkdir(os.path.join(self.local_root, "tmp"))
-        rfile_tgz = "%s/%s_*_download.tar.gz" % (
+        rfile_tgz = "{}/{}_*_download.tar.gz".format(
             self.remote_root,
             submission.submission_hash,
         )
@@ -173,7 +178,7 @@ def download(
                         raise RuntimeError("do not find download file " + rfile)
                 else:
                     if os.path.exists(lfile):
-                        dlog.info("find existing %s, replacing by %s" % (lfile, rfile))
+                        dlog.info(f"find existing {lfile}, replacing by {rfile}")
                         if os.path.isdir(lfile):
                             shutil.rmtree(lfile, ignore_errors=True)
                         elif os.path.isfile(lfile):
@@ -206,7 +211,7 @@ def download(
                     raise RuntimeError("do not find download file " + rfile)
             else:
                 if os.path.exists(lfile):
-                    dlog.info("find existing %s, replacing by %s" % (lfile, rfile))
+                    dlog.info(f"find existing {lfile}, replacing by {rfile}")
                     if os.path.isdir(lfile):
                         shutil.rmtree(lfile, ignore_errors=True)
                     elif os.path.isfile(lfile):
@@ -217,7 +222,7 @@ def download(
         shutil.rmtree(gz_dir, ignore_errors=True)
 
     def check_file_exists(self, fname):
-        """check whether the given file exists, often used in checking whether the belonging job has finished
+        """Check whether the given file exists, often used in checking whether the belonging job has finished.
 
         Parameters
         ----------