Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Only load and parse pollfile if it has changed since last loading #327

Merged
merged 9 commits into from
Aug 29, 2024
Merged

Only load and parse pollfile if it has changed since last loading #327

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3961684
Add returned default settings to docstring
johannaengland Aug 13, 2024
acf11ca
Add pollfile_mtime attribute to zino state
johannaengland Aug 2, 2024
de4af43
Only read pollfile if changed since last read
johannaengland Aug 2, 2024
d3ee7a0
Rephrase zino.toml to mention reloading only when changed
johannaengland Aug 2, 2024
a4e0ccd
Add news fragment
johannaengland Aug 2, 2024
5987e68
Catch OSError when reading pollfile
johannaengland Aug 2, 2024
b98c138
Remove noqa
johannaengland Aug 21, 2024
4999221
Remove YESTERDAY constant
johannaengland Aug 21, 2024
76cdb5b
Make pollfile_mtime unix timestamp
johannaengland Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 16 additions & 2 deletions src/zino/scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
import asyncio
import logging
import operator
from datetime import datetime, timedelta
import pathlib
from datetime import datetime, timedelta, timezone
from typing import Sequence, Set, Tuple

from apscheduler.executors.asyncio import AsyncIOExecutor
Expand Down Expand Up @@ -41,10 +42,21 @@ def get_scheduler() -> AsyncIOScheduler:

@log_time_spent()
def load_polldevs(polldevs_conf: str) -> Tuple[Set, Set, Set, dict[str, str]]:
"""Loads pollfile into process state.
"""Loads pollfile into process state if it was changed since the last time it
was loaded

:returns: A tuple of (new_devices, deleted_devices, changed_devices and a dictionary of default settings)
johannaengland marked this conversation as resolved.
Show resolved Hide resolved
"""
try:
st_mtime = pathlib.Path(polldevs_conf).stat().st_mtime
modified_time = datetime.fromtimestamp(st_mtime, tz=timezone.utc)
except OSError as e: # noqa
johannaengland marked this conversation as resolved.
Show resolved Hide resolved
_log.error(e)
return set(), set(), set(), dict()

if modified_time == state.pollfile_mtime:
return set(), set(), set(), dict()

try:
devices, defaults = read_polldevs(polldevs_conf)
except InvalidConfiguration as error:
Expand All @@ -68,6 +80,8 @@ def load_polldevs(polldevs_conf: str) -> Tuple[Set, Set, Set, dict[str, str]]:
for device in deleted_devices:
del state.polldevs[device]

state.pollfile_mtime = modified_time

return new_devices, deleted_devices, changed_devices, defaults


Expand Down
63 changes: 58 additions & 5 deletions tests/scheduler_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import logging
from datetime import timedelta
from unittest.mock import Mock, patch

import pytest
from apscheduler.jobstores.base import JobLookupError

from zino import scheduler
from zino.state import ZinoState
from zino.time import now

YESTERDAY = last_run_time = now() - timedelta(days=1)
johannaengland marked this conversation as resolved.
Show resolved Hide resolved


class TestLoadPolldevs:
@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_all_new_devices_on_first_run(self, polldevs_conf):
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_conf)
Expand All @@ -18,37 +23,46 @@ def test_should_return_all_new_devices_on_first_run(self, polldevs_conf):
assert not changed_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_defaults_on_first_run(self, polldevs_conf):
_, _, _, defaults = scheduler.load_polldevs(polldevs_conf)
assert len(defaults) > 0
assert "interval" in defaults

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_deleted_devices_on_second_run(self, polldevs_conf, polldevs_conf_with_single_router):
scheduler.load_polldevs(polldevs_conf)
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_conf_with_single_router)

# This needs to be patched since the mtime of the two conf fixtures is the same
with patch("zino.state.pollfile_mtime", YESTERDAY):
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_conf_with_single_router)

assert not new_devices
assert len(deleted_devices) > 0
assert not changed_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_no_new_or_deleted_devices_on_invalid_configuration(self, invalid_polldevs_conf):
def test__or_deleted_devices_on_invalid_configuration(self, invalid_polldevs_conf):
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(invalid_polldevs_conf)
assert not new_devices
assert not deleted_devices
assert not changed_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_log_error_on_invalid_configuration(self, caplog, invalid_polldevs_conf):
with caplog.at_level(logging.ERROR):
scheduler.load_polldevs(invalid_polldevs_conf)
assert "'lalala' is not a valid configuration line" in caplog.text

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_changed_defaults(self, polldevs_conf, tmp_path):
polldevs_with_changed_defaults = tmp_path.joinpath("changed-defaults-polldevs.cf")
Expand All @@ -69,10 +83,14 @@ def test_should_return_changed_defaults(self, polldevs_conf, tmp_path):
)

_, _, _, defaults = scheduler.load_polldevs(polldevs_conf)
_, _, _, changed_defaults = scheduler.load_polldevs(polldevs_with_changed_defaults)

# This needs to be patched since the mtime of the two conf fixtures is the same
with patch("zino.state.pollfile_mtime", YESTERDAY):
_, _, _, changed_defaults = scheduler.load_polldevs(polldevs_with_changed_defaults)
assert defaults != changed_defaults

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_changed_devices_on_changed_defaults(self, polldevs_conf, tmp_path):
polldevs_with_changed_defaults = tmp_path.joinpath("changed-defaults-polldevs.cf")
Expand All @@ -93,12 +111,17 @@ def test_should_return_changed_devices_on_changed_defaults(self, polldevs_conf,
)

scheduler.load_polldevs(polldevs_conf)
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_with_changed_defaults)

# This needs to be patched since the mtime of the two conf fixtures is the same
with patch("zino.state.pollfile_mtime", YESTERDAY):
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_with_changed_defaults)

assert not new_devices
assert not deleted_devices
assert changed_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_changed_devices_on_changed_interval(self, polldevs_conf, tmp_path):
polldevs_with_changed_defaults = tmp_path.joinpath("changed-interval-polldevs.cf")
Expand All @@ -120,14 +143,44 @@ def test_should_return_changed_devices_on_changed_interval(self, polldevs_conf,
)

scheduler.load_polldevs(polldevs_conf)
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_with_changed_defaults)

# This needs to be patched since the mtime of the two conf fixtures is the same
with patch("zino.state.pollfile_mtime", YESTERDAY):
new_devices, deleted_devices, changed_devices, _ = scheduler.load_polldevs(polldevs_with_changed_defaults)

assert not new_devices
assert not deleted_devices
assert changed_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_no_new_or_deleted_devices_on_unchanged_configuration(self, polldevs_conf):
scheduler.load_polldevs(polldevs_conf)
new_devices, deleted_devices, _, _ = scheduler.load_polldevs(polldevs_conf)
assert not new_devices
assert not deleted_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_return_no_new_or_deleted_devices_on_non_existent_pollfile(self, tmp_path):
new_devices, deleted_devices, _, _ = scheduler.load_polldevs(tmp_path / "non-existent-polldev.cf")
assert not new_devices
assert not deleted_devices

@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_log_error_on_non_existent_pollfile(self, caplog, tmp_path):
with caplog.at_level(logging.ERROR):
scheduler.load_polldevs(tmp_path / "non-existent-polldev.cf")
assert "No such file or directory" in caplog.text


class TestScheduleNewDevices:
@patch("zino.state.polldevs", dict())
@patch("zino.state.pollfile_mtime", None)
@patch("zino.state.state", ZinoState())
def test_should_schedule_jobs_for_new_devices(self, polldevs_conf, mocked_scheduler):
new_devices, _, _, _ = scheduler.load_polldevs(polldevs_conf)
Expand Down