Merge pull request #199 from mraspaud/fix-utcnow

Replace utcnow (and fix style along the way)

bin/remove_it.py

@@ -22,42 +22,47 @@
 
 """Remove files, and send messages about it."""
 
-from configparser import RawConfigParser, NoOptionError
-from datetime import datetime, timedelta
-from glob import glob
-import os
-import time
 import argparse
+import datetime as dt
+import getpass
 import logging
 import logging.handlers
-import getpass
+import os
 import socket
+import time
+from configparser import NoOptionError, RawConfigParser
+from glob import glob
 
 LOGGER = logging.getLogger("remove_it")
 
 try:
-    from posttroll.publisher import Publish
     from posttroll.message import Message
+    from posttroll.publisher import Publish
 except ImportError:
 
     class Publish(object):
+        """A fake publish in case posttroll isn't installed."""
 
         def __enter__(self):
+            """Fake enter."""
             return self
 
         def __exit__(self, etype, value, traceback):
-            pass
+            """Fake exit."""
 
         def send(self, msg):
-            pass
+            """Fake send."""
 
     def Message(*args, **kwargs):
+        """Fake message object in case posttroll isn't installed."""
         del args, kwargs
 
 
 class BufferingSMTPHandler(logging.handlers.BufferingHandler):
+    """A logging handler for emails."""
 
     def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity):
+        """Set up the handler."""
         logging.handlers.BufferingHandler.__init__(self, capacity)
         self.mailhost = mailhost
         self.mailport = None
@@ -68,6 +73,7 @@ def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity):
             logging.Formatter("[%(asctime)s %(levelname)-5s] %(message)s"))
 
     def flush(self):
+        """Flush the messages."""
         if len(self.buffer) > 0:
             try:
                 import smtplib
@@ -88,6 +94,7 @@ def flush(self):
 
 
 def parse_args():
+    """Parse the command line arguments."""
     parser = argparse.ArgumentParser()
     parser.add_argument("configuration_file",
                         help="the configuration file to use")
@@ -114,6 +121,7 @@ def parse_args():
 
 
 def setup_logger(args):
+    """Set up a logger."""
     global LOGGER
     LOGGER = logging.getLogger("remove_it")
 
@@ -137,6 +145,7 @@ def setup_logger(args):
 
 
 def setup_mailing(args, conf, info):
+    """Set up the mailing handler."""
     if args.mail:
         try:
             mailhandler = BufferingSMTPHandler(
@@ -153,6 +162,7 @@ def setup_mailing(args, conf, info):
 
 
 def get_config_items(args, conf):
+    """Get the configuration items."""
     config_items = []
 
     if args.config_item:
@@ -169,6 +179,7 @@ def get_config_items(args, conf):
 
 
 def remove_file(filename, pub):
+    """Remove one file or directory."""
     try:
         if os.path.isdir(filename):
             if not os.listdir(filename):
@@ -180,14 +191,17 @@ def remove_file(filename, pub):
             msg = Message("deletion", "del", {"uri": filename})
             pub.send(str(msg))
             LOGGER.debug("Removed %s", filename)
-    except (IOError, OSError) as err:
+    except FileNotFoundError:
+        LOGGER.debug("File already removed.")
+    except OSError as err:
         LOGGER.warning("Can't remove %s: %s", filename,
                        str(err))
         return False
     return True
 
 
 def clean_dir(pub, ref_time, pathname, is_dry_run):
+    """Clean up a directory."""
     section_files = 0
     section_size = 0
     LOGGER.info("Cleaning %s", pathname)
@@ -201,7 +215,7 @@ def clean_dir(pub, ref_time, pathname, is_dry_run):
             LOGGER.warning("Couldn't lstat path=%s", str(filename))
             continue
 
-        if datetime.fromtimestamp(stat.st_ctime) < ref_time:
+        if dt.datetime.fromtimestamp(stat.st_ctime) < ref_time:
             was_removed = False
             if not is_dry_run:
                 was_removed = remove_file(filename, pub)
@@ -215,6 +229,7 @@ def clean_dir(pub, ref_time, pathname, is_dry_run):
 
 
 def clean_section(pub, section, conf, is_dry_run=True):
+    """Clean up according to the configuration section."""
     section_files = 0
     section_size = 0
     info = dict(conf.items(section))
@@ -231,7 +246,7 @@ def clean_section(pub, section, conf, is_dry_run=True):
             kws[key] = int(info[key])
         except KeyError:
             pass
-    ref_time = datetime.utcnow() - timedelta(**kws)
+    ref_time = dt.datetime.now(dt.timezone.utc) - dt.timedelta(**kws)
 
     for template in templates:
         pathname = os.path.join(base_dir, template)
@@ -243,6 +258,7 @@ def clean_section(pub, section, conf, is_dry_run=True):
 
 
 def run(args, conf):
+    """Run the remove_it tool."""
     config_items = get_config_items(args, conf)
     LOGGER.debug("Setting up posttroll connection...")
     with Publish("remover") as pub:
@@ -261,6 +277,7 @@ def run(args, conf):
 
 
 def main():
+    """Parse args and run the remove_it tool."""
     conf = RawConfigParser()
 
     args = parse_args()

trollmoves/server.py

@@ -64,7 +64,7 @@
 
 file_cache = deque(maxlen=61000)
 file_cache_lock = Lock()
-START_TIME = datetime.datetime.utcnow()
+START_TIME = datetime.datetime.now(datetime.timezone.utc)
 
 CONNECTION_CONFIG_ITEMS = ["connection_uptime", "ssh_key_filename", "ssh_connection_timeout", "ssh_private_key_file"]
 
@@ -204,7 +204,7 @@ def ack(self, message):
 
     def info(self, message):
         """Collect information from file cache to message."""
-        uptime = datetime.datetime.utcnow() - START_TIME
+        uptime = datetime.datetime.now(datetime.timezone.utc) - START_TIME
         files, max_count = _collect_cached_files(message)
 
         return Message(message.subject, "info", data={"files": files, "max_count": max_count, "uptime": str(uptime)})
@@ -629,7 +629,8 @@ def _form_connection_parameters_dict(original):
         if key in CONNECTION_CONFIG_ITEMS:
             warnings.warn(
                 f"Consider using connection_parameters__{key} instead of {key}.",
-                category=UserWarning, stacklevel=2)
+                category=UserWarning,
+                stacklevel=2)
             res["connection_parameters"][key] = original[key]
             del res[key]
     return res

trollmoves/tests/functional/test_move_it_server_client.py

@@ -1,9 +1,9 @@
 """Acceptance tests for move it server/client."""
 
+import datetime as dt
 import os
 import socket
 import time
-from datetime import datetime
 from pathlib import Path
 from threading import Thread
 
@@ -132,7 +132,7 @@ def start_move_it_client(client):
 def create_new_file(source_dir):
     """Create a new file in source_dir."""
     pattern = "H-000-%Y%m%d%H%M-__"
-    filename = datetime.utcnow().strftime(pattern)
+    filename = dt.datetime.now(dt.timezone.utc).strftime(pattern)
     path = Path(source_dir / filename)
     path.write_bytes(b"Very Important Satellite Data")
     return filename
@@ -251,7 +251,7 @@ def start_move_it_server_with_untarring(server_without_request_port_and_untarrin
 def create_new_tar_file(source_dir):
     """Create a new file in source_dir."""
     pattern = "H-000-%Y%m%d%H%M-__"
-    filename = datetime.utcnow().strftime(pattern)
+    filename = dt.datetime.now(dt.timezone.utc).strftime(pattern)
     path = source_dir / filename
     path.write_bytes(b"Very Important Satellite Data")
     tarfilename = source_dir / (filename + ".tar")