Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logging to fetcher #201

Merged
merged 3 commits into from
Jun 17, 2024
Merged

Add logging to fetcher #201

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e94d685
Add logging to fetcher
mraspaud May 30, 2024
809af1e
Merge branch 'main' into add-logging-to-fetcher
mraspaud May 30, 2024
05f1b38
Improve fetcher logs test
mraspaud May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions trollmoves/fetcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,13 @@ def fetch_from_subscriber(destination, subscriber_config, publisher_config):
for message in sub.recv():
if message.type != "file":
continue
logger.debug(f"Fetching from {str(message)}")
downloaded_file = fetch_from_message(message, destination)
message.data.pop("filesystem", None)
message.data.pop("path", None)
message.data["uri"] = downloaded_file.as_uri()
pub.send(str(message))
logger.debug(f"Published {str(message)}")


def cli(args=None):
Expand Down
14 changes: 11 additions & 3 deletions trollmoves/tests/test_fetcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def test_fetch_message_with_uri(tmp_path):

def test_fetch_message_logs(tmp_path, caplog):
"""Test fetch_message logs."""
caplog.set_level("INFO")
caplog.set_level("DEBUG")

uid = "IVCDB_j02_d20240419_t1114110_e1115356_b07465_c20240419113435035578_cspp_dev.h5"
sdr_file = tmp_path / "sdr" / uid
Expand All @@ -153,8 +153,16 @@ def test_fetch_message_logs(tmp_path, caplog):
dest_path2 = tmp_path / "dest2"
dest_path2.mkdir()

downloaded_file = fetch_from_message(Message(rawstr=msg), dest_path2)
assert str(downloaded_file) in caplog.text
subscriber_settings = dict(nameserver=False, addresses=["ipc://bla"])
publisher_settings = dict(nameservers=False, port=1979)

with patched_publisher() as messages:
with patched_subscriber_recv([Message(rawstr=msg)]):
fetch_from_subscriber(dest_path2, subscriber_settings, publisher_settings)

assert str(msg) in caplog.text
assert str(dest_path2 / uid) in caplog.text
assert f"Published {messages[0]}" in caplog.text


def test_subscribe_and_fetch(tmp_path):
Expand Down