From 72637801e41dd64489faeb96c3eccdebceeb7b9c Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 27 May 2024 09:01:45 +0200 Subject: [PATCH 1/3] Replace utcnow (and fix style along the way) --- bin/remove_it.py | 41 +++++++++++++------ trollmoves/server.py | 7 ++-- .../functional/test_move_it_server_client.py | 6 +-- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/bin/remove_it.py b/bin/remove_it.py index 229a50b6..3a0fa751 100644 --- a/bin/remove_it.py +++ b/bin/remove_it.py @@ -22,42 +22,47 @@ """Remove files, and send messages about it.""" -from configparser import RawConfigParser, NoOptionError -from datetime import datetime, timedelta -from glob import glob -import os -import time import argparse +import datetime as dt +import getpass import logging import logging.handlers -import getpass +import os import socket +import time +from configparser import NoOptionError, RawConfigParser +from glob import glob LOGGER = logging.getLogger("remove_it") try: - from posttroll.publisher import Publish from posttroll.message import Message + from posttroll.publisher import Publish except ImportError: class Publish(object): + """A fake publish in case posttroll isn't installed.""" def __enter__(self): + """Fake enter.""" return self def __exit__(self, etype, value, traceback): - pass + """Fake exit.""" def send(self, msg): - pass + """Fake send.""" def Message(*args, **kwargs): + """Fake message object in case posttroll isn't installed.""" del args, kwargs class BufferingSMTPHandler(logging.handlers.BufferingHandler): + """A logging handler for emails.""" def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity): + """Set up the handler.""" logging.handlers.BufferingHandler.__init__(self, capacity) self.mailhost = mailhost self.mailport = None @@ -68,6 +73,7 @@ def __init__(self, mailhost, fromaddr, toaddrs, subject, capacity): logging.Formatter("[%(asctime)s %(levelname)-5s] %(message)s")) def flush(self): + """Flush the messages.""" if len(self.buffer) > 0: try: import smtplib @@ -88,6 +94,7 @@ def flush(self): def parse_args(): + """Parse the command line arguments.""" parser = argparse.ArgumentParser() parser.add_argument("configuration_file", help="the configuration file to use") @@ -114,6 +121,7 @@ def parse_args(): def setup_logger(args): + """Set up a logger.""" global LOGGER LOGGER = logging.getLogger("remove_it") @@ -137,6 +145,7 @@ def setup_logger(args): def setup_mailing(args, conf, info): + """Set up the mailing handler.""" if args.mail: try: mailhandler = BufferingSMTPHandler( @@ -153,6 +162,7 @@ def setup_mailing(args, conf, info): def get_config_items(args, conf): + """Get the configuration items.""" config_items = [] if args.config_item: @@ -169,6 +179,7 @@ def get_config_items(args, conf): def remove_file(filename, pub): + """Remove one file or directory.""" try: if os.path.isdir(filename): if not os.listdir(filename): @@ -180,7 +191,9 @@ def remove_file(filename, pub): msg = Message("deletion", "del", {"uri": filename}) pub.send(str(msg)) LOGGER.debug("Removed %s", filename) - except (IOError, OSError) as err: + except FileNotFoundError: + LOGGER.debug("File already removed.") + except OSError as err: LOGGER.warning("Can't remove %s: %s", filename, str(err)) return False @@ -188,6 +201,7 @@ def remove_file(filename, pub): def clean_dir(pub, ref_time, pathname, is_dry_run): + """Clean up a directory.""" section_files = 0 section_size = 0 LOGGER.info("Cleaning %s", pathname) @@ -201,7 +215,7 @@ def clean_dir(pub, ref_time, pathname, is_dry_run): LOGGER.warning("Couldn't lstat path=%s", str(filename)) continue - if datetime.fromtimestamp(stat.st_ctime) < ref_time: + if dt.datetime.fromtimestamp(stat.st_ctime) < ref_time: was_removed = False if not is_dry_run: was_removed = remove_file(filename, pub) @@ -215,6 +229,7 @@ def clean_dir(pub, ref_time, pathname, is_dry_run): def clean_section(pub, section, conf, is_dry_run=True): + """Clean up according to the configuration section.""" section_files = 0 section_size = 0 info = dict(conf.items(section)) @@ -231,7 +246,7 @@ def clean_section(pub, section, conf, is_dry_run=True): kws[key] = int(info[key]) except KeyError: pass - ref_time = datetime.utcnow() - timedelta(**kws) + ref_time = dt.datetime.now(dt.UTC) - dt.timedelta(**kws) for template in templates: pathname = os.path.join(base_dir, template) @@ -243,6 +258,7 @@ def clean_section(pub, section, conf, is_dry_run=True): def run(args, conf): + """Run the remove_it tool.""" config_items = get_config_items(args, conf) LOGGER.debug("Setting up posttroll connection...") with Publish("remover") as pub: @@ -261,6 +277,7 @@ def run(args, conf): def main(): + """Parse args and run the remove_it tool.""" conf = RawConfigParser() args = parse_args() diff --git a/trollmoves/server.py b/trollmoves/server.py index eafc7695..48f93aa5 100644 --- a/trollmoves/server.py +++ b/trollmoves/server.py @@ -64,7 +64,7 @@ file_cache = deque(maxlen=61000) file_cache_lock = Lock() -START_TIME = datetime.datetime.utcnow() +START_TIME = datetime.datetime.now(datetime.UTC) CONNECTION_CONFIG_ITEMS = ["connection_uptime", "ssh_key_filename", "ssh_connection_timeout", "ssh_private_key_file"] @@ -204,7 +204,7 @@ def ack(self, message): def info(self, message): """Collect information from file cache to message.""" - uptime = datetime.datetime.utcnow() - START_TIME + uptime = datetime.datetime.now(datetime.UTC) - START_TIME files, max_count = _collect_cached_files(message) return Message(message.subject, "info", data={"files": files, "max_count": max_count, "uptime": str(uptime)}) @@ -649,7 +649,8 @@ def _form_connection_parameters_dict(original): if key in CONNECTION_CONFIG_ITEMS: warnings.warn( f"Consider using connection_parameters__{key} instead of {key}.", - category=UserWarning) + category=UserWarning, + stacklevel=2) res["connection_parameters"][key] = original[key] del res[key] return res diff --git a/trollmoves/tests/functional/test_move_it_server_client.py b/trollmoves/tests/functional/test_move_it_server_client.py index 9911d137..502a13ed 100644 --- a/trollmoves/tests/functional/test_move_it_server_client.py +++ b/trollmoves/tests/functional/test_move_it_server_client.py @@ -1,9 +1,9 @@ """Acceptance tests for move it server/client.""" +import datetime as dt import os import socket import time -from datetime import datetime from pathlib import Path from threading import Thread @@ -132,7 +132,7 @@ def start_move_it_client(client): def create_new_file(source_dir): """Create a new file in source_dir.""" pattern = "H-000-%Y%m%d%H%M-__" - filename = datetime.utcnow().strftime(pattern) + filename = dt.datetime.now(dt.UTC).strftime(pattern) path = Path(source_dir / filename) path.write_bytes(b"Very Important Satellite Data") return filename @@ -251,7 +251,7 @@ def start_move_it_server_with_untarring(server_without_request_port_and_untarrin def create_new_tar_file(source_dir): """Create a new file in source_dir.""" pattern = "H-000-%Y%m%d%H%M-__" - filename = datetime.utcnow().strftime(pattern) + filename = dt.datetime.now(dt.UTC).strftime(pattern) path = source_dir / filename path.write_bytes(b"Very Important Satellite Data") tarfilename = source_dir / (filename + ".tar") From 2834106cb0daadd0381fedb08aea817869e17c26 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 27 May 2024 09:08:16 +0200 Subject: [PATCH 2/3] Use timezone.utc instead of UTC for py3.10 compatibility --- bin/remove_it.py | 2 +- trollmoves/server.py | 4 ++-- trollmoves/tests/functional/test_move_it_server_client.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/remove_it.py b/bin/remove_it.py index 3a0fa751..e1c63785 100644 --- a/bin/remove_it.py +++ b/bin/remove_it.py @@ -246,7 +246,7 @@ def clean_section(pub, section, conf, is_dry_run=True): kws[key] = int(info[key]) except KeyError: pass - ref_time = dt.datetime.now(dt.UTC) - dt.timedelta(**kws) + ref_time = dt.datetime.now(dt.timezone.utc) - dt.timedelta(**kws) for template in templates: pathname = os.path.join(base_dir, template) diff --git a/trollmoves/server.py b/trollmoves/server.py index 48f93aa5..de7bbfbe 100644 --- a/trollmoves/server.py +++ b/trollmoves/server.py @@ -64,7 +64,7 @@ file_cache = deque(maxlen=61000) file_cache_lock = Lock() -START_TIME = datetime.datetime.now(datetime.UTC) +START_TIME = datetime.datetime.now(datetime.timezone.utc) CONNECTION_CONFIG_ITEMS = ["connection_uptime", "ssh_key_filename", "ssh_connection_timeout", "ssh_private_key_file"] @@ -204,7 +204,7 @@ def ack(self, message): def info(self, message): """Collect information from file cache to message.""" - uptime = datetime.datetime.now(datetime.UTC) - START_TIME + uptime = datetime.datetime.now(datetime.timezone.utc) - START_TIME files, max_count = _collect_cached_files(message) return Message(message.subject, "info", data={"files": files, "max_count": max_count, "uptime": str(uptime)}) diff --git a/trollmoves/tests/functional/test_move_it_server_client.py b/trollmoves/tests/functional/test_move_it_server_client.py index 502a13ed..58710cbf 100644 --- a/trollmoves/tests/functional/test_move_it_server_client.py +++ b/trollmoves/tests/functional/test_move_it_server_client.py @@ -132,7 +132,7 @@ def start_move_it_client(client): def create_new_file(source_dir): """Create a new file in source_dir.""" pattern = "H-000-%Y%m%d%H%M-__" - filename = dt.datetime.now(dt.UTC).strftime(pattern) + filename = dt.datetime.now(dt.timezone.utc).strftime(pattern) path = Path(source_dir / filename) path.write_bytes(b"Very Important Satellite Data") return filename @@ -251,7 +251,7 @@ def start_move_it_server_with_untarring(server_without_request_port_and_untarrin def create_new_tar_file(source_dir): """Create a new file in source_dir.""" pattern = "H-000-%Y%m%d%H%M-__" - filename = dt.datetime.now(dt.UTC).strftime(pattern) + filename = dt.datetime.now(dt.timezone.utc).strftime(pattern) path = source_dir / filename path.write_bytes(b"Very Important Satellite Data") tarfilename = source_dir / (filename + ".tar") From 1ed636f247c8db532154b80b7fbb390a20fd9050 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Mon, 27 May 2024 11:19:38 +0200 Subject: [PATCH 3/3] Fix funky merge --- trollmoves/server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/trollmoves/server.py b/trollmoves/server.py index 19aa1d20..8d2938e2 100644 --- a/trollmoves/server.py +++ b/trollmoves/server.py @@ -629,12 +629,8 @@ def _form_connection_parameters_dict(original): if key in CONNECTION_CONFIG_ITEMS: warnings.warn( f"Consider using connection_parameters__{key} instead of {key}.", -<<<<<<< fix-utcnow category=UserWarning, stacklevel=2) -======= - category=UserWarning, stacklevel=2) ->>>>>>> main res["connection_parameters"][key] = original[key] del res[key] return res