diff --git a/trollmoves/tests/test_fetcher.py b/trollmoves/tests/test_fetcher.py index a9d23a4..23100ea 100644 --- a/trollmoves/tests/test_fetcher.py +++ b/trollmoves/tests/test_fetcher.py @@ -5,6 +5,7 @@ from zipfile import ZipFile import fsspec +import yaml from posttroll.message import Message from posttroll.testing import patched_publisher, patched_subscriber_recv @@ -125,15 +126,11 @@ def test_fetch_message_with_filesystem(tmp_path): def test_fetch_message_with_uri(tmp_path): """Test fetch_message can use uri.""" uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" - + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) msg = ('pytroll://segment/viirs/l1b/ file a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}"' '}') - - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}"' '}') dest_path2 = tmp_path / "dest2" dest_path2.mkdir() @@ -147,15 +144,11 @@ def test_fetch_message_logs(tmp_path, caplog): caplog.set_level("INFO") uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" - + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) msg = ('pytroll://segment/viirs/l1b/ file a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}"' '}') - - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}"' '}') dest_path2 = tmp_path / "dest2" dest_path2.mkdir() @@ -167,17 +160,14 @@ def test_fetch_message_logs(tmp_path, caplog): def test_subscribe_and_fetch(tmp_path): """Test subscribe and fetch.""" uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) msg = ('pytroll://segment/viirs/l1b/ file a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}", "path": "{str(tmp_path)}/sdr/{uid}", ' + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}", "path": "{str(sdr_file)}", ' '"filesystem": {"cls": "fsspec.implementations.local.LocalFileSystem", "protocol": "file", "args": []}}') - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") - dest_path2 = tmp_path / "dest2" dest_path2.mkdir() @@ -201,32 +191,19 @@ def test_fetcher_cli(tmp_path): """Test the fetcher command-line interface.""" config_file = tmp_path / "config.yaml" destination = tmp_path / "destination" - destination.mkdir() - subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) - publisher_settings = dict(nameservers=False, port=1979) - config = dict(destination=str(destination), - subscriber_config=subscriber_settings, - publisher_config=publisher_settings) - - import yaml - with open(config_file, "w") as fd: - fd.write(yaml.dump(config)) + create_config_file(destination, config_file) uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) msg = ('pytroll://segment/viirs/l1b/ file a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}", "path": "{str(tmp_path)}/sdr/{uid}", ' + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}", "path": "{str(sdr_file)}", ' '"filesystem": {"cls": "fsspec.implementations.local.LocalFileSystem", "protocol": "file", "args": []}}') - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") - with patched_publisher() as messages: with patched_subscriber_recv([Message(rawstr=msg)]): - from trollmoves.fetcher import cli cli([str(config_file)]) @@ -243,29 +220,17 @@ def test_fetcher_does_not_try_to_fetch_non_file_messages(tmp_path): """Test fetcher does not try to fetch non-file messages.""" config_file = tmp_path / "config.yaml" destination = tmp_path / "destination" - destination.mkdir() - subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) - publisher_settings = dict(nameservers=False, port=1979) - config = dict(destination=str(destination), - subscriber_config=subscriber_settings, - publisher_config=publisher_settings) - - import yaml - with open(config_file, "w") as fd: - fd.write(yaml.dump(config)) + create_config_file(destination, config_file) uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) msg = ('pytroll://segment/viirs/l1b/ info a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}", "path": "{str(tmp_path)}/sdr/{uid}", ' + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}", "path": "{str(sdr_file)}", ' '"filesystem": {"cls": "fsspec.implementations.local.LocalFileSystem", "protocol": "file", "args": []}}') - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") - with patched_publisher() as messages: with patched_subscriber_recv([Message(rawstr=msg)]): @@ -280,31 +245,56 @@ def test_fetcher_uses_log_config(tmp_path): """Test fetcher uses log config.""" config_file = tmp_path / "config.yaml" destination = tmp_path / "destination" + create_config_file(destination, config_file) + + uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" + sdr_file = tmp_path / "sdr" / uid + create_data_file(sdr_file) + + msg = ('pytroll://segment/viirs/l1b/ info a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' + 'application/json {"sensor": "viirs", ' + f'"uid": "{uid}", "uri": "file://{str(sdr_file)}", "path": "{str(sdr_file)}", ' + '"filesystem": {"cls": "fsspec.implementations.local.LocalFileSystem", "protocol": "file", "args": []}}') + + log_config_file = tmp_path / "log_config.yaml" + handler_name = "console123" + create_log_config(log_config_file, handler_name) + + with patched_publisher(): + with patched_subscriber_recv([Message(rawstr=msg)]): + + from trollmoves.fetcher import cli + cli([str(config_file), "-c", str(log_config_file)]) + + root = logging.getLogger() + assert len(root.handlers) == 1 + assert root.handlers[0].name == handler_name + + +def create_config_file(destination, config_file): + """Create a configuration file.""" destination.mkdir() subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"]) publisher_settings = dict(nameservers=False, port=1979) config = dict(destination=str(destination), subscriber_config=subscriber_settings, publisher_config=publisher_settings) - - import yaml with open(config_file, "w") as fd: fd.write(yaml.dump(config)) - uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5" + return config - msg = ('pytroll://segment/viirs/l1b/ info a001673@c22519.ad.smhi.se 2024-04-19T11:35:00.487388 v1.01 ' - 'application/json {"sensor": "viirs", ' - f'"uid": "{uid}", "uri": "file://{str(tmp_path)}/sdr/{uid}", "path": "{str(tmp_path)}/sdr/{uid}", ' - '"filesystem": {"cls": "fsspec.implementations.local.LocalFileSystem", "protocol": "file", "args": []}}') - sdr = tmp_path / "sdr" - sdr.mkdir() - with open(sdr / uid, "w") as fd: - fd.write("viirs data") +def create_data_file(path): + """Create a data file.""" + path.parent.mkdir() - log_config_file = tmp_path / "log_config.yaml" - handler_name = "console123" + with open(path, "w") as fd: + fd.write("data") + + +def create_log_config(log_config_file, handler_name): + """Create a log config file.""" log_config = { "version": 1, "handlers": { @@ -322,13 +312,3 @@ def test_fetcher_uses_log_config(tmp_path): } with open(log_config_file, "w") as fd: fd.write(yaml.dump(log_config)) - - with patched_publisher(): - with patched_subscriber_recv([Message(rawstr=msg)]): - - from trollmoves.fetcher import cli - cli([str(config_file), "-c", str(log_config_file)]) - - root = logging.getLogger() - assert len(root.handlers) == 1 - assert root.handlers[0].name == handler_name