Skip to content

Commit

Permalink
Merge pull request #223 from pnuu/fix-trollsched-usage-when-not-insta…
Browse files Browse the repository at this point in the history 
…lled

Fix usage when trollsched is not installed
  • Loading branch information
@mraspaud
mraspaud authored Nov 18, 2024
2 parents beb12c6 + 49bafdc commit c57635b
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 47 deletions.
15 changes: 10 additions & 5 deletions trollflow2/plugins/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -639,8 +639,8 @@ def _check_overall_coverage_for_area(
Helper for covers().
"""
area_path = "/product_list/areas/%s" % area
cov = get_scene_coverage(platform_name, start_time, end_time,
sensor, area)
cov = _get_scene_coverage(platform_name, start_time, end_time,
sensor, area)
product_list['product_list']['areas'][area]['area_coverage_percent'] = cov
if cov < min_coverage:
logger.info(
Expand All @@ -654,7 +654,7 @@ def _check_overall_coverage_for_area(
f"{min_coverage:.2f}% - Carry on with {area:s}")


def get_scene_coverage(platform_name, start_time, end_time, sensor, area_id):
def _get_scene_coverage(platform_name, start_time, end_time, sensor, area_id):
"""Get scene area coverage in percentages."""
overpass = Pass(platform_name, start_time, end_time, instrument=sensor)
area_def = get_area_def(area_id)
Expand Down Expand Up @@ -968,6 +968,11 @@ def check_valid_data_fraction(job):
"""
logger.info("Checking valid data fraction.")

if get_twilight_poly is None:
logger.error("Trollsched import failed, calculation of valid data fraction not possible")
logger.info("Keeping all products")
return

exp_cov = {}
# As stated, this will trigger a computation. To prevent computing
# multiple times, we should persist everything that needs to be persisted,
Expand Down Expand Up @@ -1028,8 +1033,8 @@ def _product_meets_min_valid_data_fraction(
end_time = prod.attrs["end_time"]
sensor = prod.attrs["sensor"]
if area_name not in exp_cov:
# get_scene_coverage uses %, convert to fraction
exp_cov[area_name] = get_scene_coverage(
# _get_scene_coverage uses %, convert to fraction
exp_cov[area_name] = _get_scene_coverage(
platform_name, start_time, end_time, sensor, area_name)/100
exp_valid = exp_cov[area_name]
if exp_valid == 0:
Expand Down
126 changes: 84 additions & 42 deletions trollflow2/tests/test_trollflow2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1353,9 +1353,9 @@ def test_covers_complains_when_multiple_sensors_are_provided(self):
"""Test that the plugin complains when multiple sensors are provided."""
from trollflow2.plugins import covers

with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \
with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \
mock.patch('trollflow2.plugins.Pass'):
get_scene_coverage.return_value = 10.0
_get_scene_coverage.return_value = 10.0
scn = _get_mocked_scene_with_properties()
job = {"product_list": self.product_list,
"input_mda": {"platform_name": "platform",
Expand All @@ -1371,9 +1371,9 @@ def test_covers_does_not_complain_when_one_sensor_is_provided_as_a_sequence(self
"""Test that the plugin complains when multiple sensors are provided."""
from trollflow2.plugins import covers

with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \
with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \
mock.patch('trollflow2.plugins.Pass'):
get_scene_coverage.return_value = 10.0
_get_scene_coverage.return_value = 10.0
scn = _get_mocked_scene_with_properties()
job = {"product_list": self.product_list,
"input_mda": {"platform_name": "platform",
Expand All @@ -1389,19 +1389,19 @@ def test_metadata_is_read_from_scene(self):
"""Test that the scene and message metadata are merged correctly."""
from trollflow2.plugins import covers

with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \
with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \
mock.patch('trollflow2.plugins.Pass'):
get_scene_coverage.return_value = 10.0
_get_scene_coverage.return_value = 10.0
scn = _get_mocked_scene_with_properties()
job = {"product_list": self.product_list,
"input_mda": {"platform_name": "platform"},
"scene": scn}
covers(job)
get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"],
scn.start_time,
scn.end_time,
list(scn.sensor_names)[0],
"omerc_bb")
_get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"],
scn.start_time,
scn.end_time,
list(scn.sensor_names)[0],
"omerc_bb")

def test_covers(self):
"""Test coverage."""
Expand Down Expand Up @@ -1435,14 +1435,14 @@ def test_covers_uses_only_one_sensor(self):
"scene": scn}
job2 = copy.deepcopy(job)

with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \
with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \
mock.patch('trollflow2.plugins.Pass'):
get_scene_coverage.return_value = 10.0
_get_scene_coverage.return_value = 10.0
covers(job)
get_scene_coverage.assert_called_with(input_mda['platform_name'],
input_mda['start_time'],
input_mda['end_time'],
'avhrr-4', 'omerc_bb')
_get_scene_coverage.assert_called_with(input_mda['platform_name'],
input_mda['start_time'],
input_mda['end_time'],
'avhrr-4', 'omerc_bb')

del job2["product_list"]["product_list"]["areas"]["euron1"]["min_coverage"]
del job2["product_list"]["product_list"]["min_coverage"]
Expand All @@ -1452,7 +1452,7 @@ def test_covers_uses_only_one_sensor(self):

def test_scene_coverage(self):
"""Test scene coverage."""
from trollflow2.plugins import get_scene_coverage
from trollflow2.plugins import _get_scene_coverage
with mock.patch('trollflow2.plugins.get_area_def') as get_area_def, \
mock.patch('trollflow2.plugins.Pass') as ts_pass:
area_coverage = mock.MagicMock()
Expand All @@ -1461,7 +1461,7 @@ def test_scene_coverage(self):
overpass.area_coverage = area_coverage
ts_pass.return_value = overpass
get_area_def.return_value = 6
res = get_scene_coverage(1, 2, 3, 4, 5)
res = _get_scene_coverage(1, 2, 3, 4, 5)
self.assertEqual(res, 100 * 0.2)
ts_pass.assert_called_with(1, 2, 3, instrument=4)
get_area_def.assert_called_with(5)
Expand Down Expand Up @@ -2174,24 +2174,13 @@ def sc_3a_3b():
return scene


def test_valid_filter(caplog, sc_3a_3b):
"""Test filter for minimum fraction of valid data."""
from trollflow2.launcher import yaml
def test_valid_filter_full_coverage(caplog, sc_3a_3b):
"""Test filter for minimum fraction of valid data with full coverage."""
from trollflow2.plugins import check_valid_data_fraction
product_list = yaml.safe_load(yaml_test3)

job = {}
job['scene'] = sc_3a_3b
job['product_list'] = product_list.copy()
job['input_mda'] = input_mda.copy()
job['resampled_scenes'] = {"euron1": sc_3a_3b}
prods = job['product_list']['product_list']['areas']['euron1']['products']
for p in ("NIR016", "IR037", "absent"):
prods[p] = {"min_valid_data_fraction": 40}
job2 = copy.deepcopy(job)
prods2 = job2['product_list']['product_list']['areas']['euron1']['products']
job, prods = _create_valid_filter_job_and_prods(sc_3a_3b)

with mock.patch("trollflow2.plugins.get_scene_coverage") as tpg, \
with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \
caplog.at_level(logging.DEBUG):
tpg.return_value = 100
check_valid_data_fraction(job)
Expand All @@ -2200,18 +2189,71 @@ def test_valid_filter(caplog, sc_3a_3b):
assert "removing NIR016 for area euron1" in caplog.text
assert "keeping IR037 for area euron1" in caplog.text
assert "product absent not found, already removed" in caplog.text
tpg.reset_mock()


def test_valid_filter_small_coverage(caplog, sc_3a_3b):
"""Test filter for minimum fraction of valid data with small coverage."""
from trollflow2.plugins import check_valid_data_fraction

job, prods = _create_valid_filter_job_and_prods(sc_3a_3b)

with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \
caplog.at_level(logging.DEBUG):
tpg.return_value = 1
check_valid_data_fraction(job2)
check_valid_data_fraction(job)
assert "inaccurate coverage estimate suspected!" in caplog.text
assert "NIR016" in prods2
assert "IR037" in prods2
tpg.reset_mock()
assert "NIR016" in prods
assert "IR037" in prods


def test_valid_filter_zero_coverage(caplog, sc_3a_3b):
"""Test filter for minimum fraction of valid data without any coverage."""
from trollflow2.plugins import check_valid_data_fraction

job, prods = _create_valid_filter_job_and_prods(sc_3a_3b)

with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \
caplog.at_level(logging.DEBUG):
tpg.return_value = 0
check_valid_data_fraction(job2)
check_valid_data_fraction(job)
assert "no expected coverage at all, removing" in caplog.text
assert "NIR016" not in prods2
assert "IR037" not in prods2
assert "NIR016" not in prods
assert "IR037" not in prods


def test_valid_filter_no_trollsched(caplog, monkeypatch, sc_3a_3b):
"""Test filter for minimum fraction of valid data with full coverage."""
# This is needed when only this test is run
monkeypatch.setattr("trollsched.spherical.get_twilight_poly", None)
# ... and this when all the tests are run
monkeypatch.setattr("trollflow2.plugins.get_twilight_poly", None)
from trollflow2.plugins import check_valid_data_fraction

job, prods = _create_valid_filter_job_and_prods(sc_3a_3b)

with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \
caplog.at_level(logging.DEBUG):
tpg.return_value = 100
check_valid_data_fraction(job)

assert "Trollsched import failed" in caplog.text
assert "Keeping all products" in caplog.text


def _create_valid_filter_job_and_prods(sc_3a_3b):
from trollflow2.launcher import yaml
product_list = yaml.safe_load(yaml_test3)
job = {}
job['scene'] = sc_3a_3b
job['product_list'] = product_list.copy()
job['input_mda'] = input_mda.copy()
job['resampled_scenes'] = {"euron1": sc_3a_3b}

prods = job['product_list']['product_list']['areas']['euron1']['products']
for p in ("NIR016", "IR037", "absent"):
prods[p] = {"min_valid_data_fraction": 40}

return job, prods


def test_persisted(sc_3a_3b):
Expand Down

0 comments on commit c57635b

Please sign in to comment.