diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 8b3e031..33f53f1 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -639,8 +639,8 @@ def _check_overall_coverage_for_area( Helper for covers(). """ area_path = "/product_list/areas/%s" % area - cov = get_scene_coverage(platform_name, start_time, end_time, - sensor, area) + cov = _get_scene_coverage(platform_name, start_time, end_time, + sensor, area) product_list['product_list']['areas'][area]['area_coverage_percent'] = cov if cov < min_coverage: logger.info( @@ -654,7 +654,7 @@ def _check_overall_coverage_for_area( f"{min_coverage:.2f}% - Carry on with {area:s}") -def get_scene_coverage(platform_name, start_time, end_time, sensor, area_id): +def _get_scene_coverage(platform_name, start_time, end_time, sensor, area_id): """Get scene area coverage in percentages.""" overpass = Pass(platform_name, start_time, end_time, instrument=sensor) area_def = get_area_def(area_id) @@ -968,6 +968,11 @@ def check_valid_data_fraction(job): """ logger.info("Checking valid data fraction.") + if get_twilight_poly is None: + logger.error("Trollsched import failed, calculation of valid data fraction not possible") + logger.info("Keeping all products") + return + exp_cov = {} # As stated, this will trigger a computation. To prevent computing # multiple times, we should persist everything that needs to be persisted, @@ -1028,8 +1033,8 @@ def _product_meets_min_valid_data_fraction( end_time = prod.attrs["end_time"] sensor = prod.attrs["sensor"] if area_name not in exp_cov: - # get_scene_coverage uses %, convert to fraction - exp_cov[area_name] = get_scene_coverage( + # _get_scene_coverage uses %, convert to fraction + exp_cov[area_name] = _get_scene_coverage( platform_name, start_time, end_time, sensor, area_name)/100 exp_valid = exp_cov[area_name] if exp_valid == 0: diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 9f362ef..7ae131b 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1353,9 +1353,9 @@ def test_covers_complains_when_multiple_sensors_are_provided(self): """Test that the plugin complains when multiple sensors are provided.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform", @@ -1371,9 +1371,9 @@ def test_covers_does_not_complain_when_one_sensor_is_provided_as_a_sequence(self """Test that the plugin complains when multiple sensors are provided.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform", @@ -1389,19 +1389,19 @@ def test_metadata_is_read_from_scene(self): """Test that the scene and message metadata are merged correctly.""" from trollflow2.plugins import covers - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 scn = _get_mocked_scene_with_properties() job = {"product_list": self.product_list, "input_mda": {"platform_name": "platform"}, "scene": scn} covers(job) - get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"], - scn.start_time, - scn.end_time, - list(scn.sensor_names)[0], - "omerc_bb") + _get_scene_coverage.assert_called_with(job["input_mda"]["platform_name"], + scn.start_time, + scn.end_time, + list(scn.sensor_names)[0], + "omerc_bb") def test_covers(self): """Test coverage.""" @@ -1435,14 +1435,14 @@ def test_covers_uses_only_one_sensor(self): "scene": scn} job2 = copy.deepcopy(job) - with mock.patch('trollflow2.plugins.get_scene_coverage') as get_scene_coverage, \ + with mock.patch('trollflow2.plugins._get_scene_coverage') as _get_scene_coverage, \ mock.patch('trollflow2.plugins.Pass'): - get_scene_coverage.return_value = 10.0 + _get_scene_coverage.return_value = 10.0 covers(job) - get_scene_coverage.assert_called_with(input_mda['platform_name'], - input_mda['start_time'], - input_mda['end_time'], - 'avhrr-4', 'omerc_bb') + _get_scene_coverage.assert_called_with(input_mda['platform_name'], + input_mda['start_time'], + input_mda['end_time'], + 'avhrr-4', 'omerc_bb') del job2["product_list"]["product_list"]["areas"]["euron1"]["min_coverage"] del job2["product_list"]["product_list"]["min_coverage"] @@ -1452,7 +1452,7 @@ def test_covers_uses_only_one_sensor(self): def test_scene_coverage(self): """Test scene coverage.""" - from trollflow2.plugins import get_scene_coverage + from trollflow2.plugins import _get_scene_coverage with mock.patch('trollflow2.plugins.get_area_def') as get_area_def, \ mock.patch('trollflow2.plugins.Pass') as ts_pass: area_coverage = mock.MagicMock() @@ -1461,7 +1461,7 @@ def test_scene_coverage(self): overpass.area_coverage = area_coverage ts_pass.return_value = overpass get_area_def.return_value = 6 - res = get_scene_coverage(1, 2, 3, 4, 5) + res = _get_scene_coverage(1, 2, 3, 4, 5) self.assertEqual(res, 100 * 0.2) ts_pass.assert_called_with(1, 2, 3, instrument=4) get_area_def.assert_called_with(5) @@ -2174,24 +2174,13 @@ def sc_3a_3b(): return scene -def test_valid_filter(caplog, sc_3a_3b): - """Test filter for minimum fraction of valid data.""" - from trollflow2.launcher import yaml +def test_valid_filter_full_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data with full coverage.""" from trollflow2.plugins import check_valid_data_fraction - product_list = yaml.safe_load(yaml_test3) - job = {} - job['scene'] = sc_3a_3b - job['product_list'] = product_list.copy() - job['input_mda'] = input_mda.copy() - job['resampled_scenes'] = {"euron1": sc_3a_3b} - prods = job['product_list']['product_list']['areas']['euron1']['products'] - for p in ("NIR016", "IR037", "absent"): - prods[p] = {"min_valid_data_fraction": 40} - job2 = copy.deepcopy(job) - prods2 = job2['product_list']['product_list']['areas']['euron1']['products'] + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) - with mock.patch("trollflow2.plugins.get_scene_coverage") as tpg, \ + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ caplog.at_level(logging.DEBUG): tpg.return_value = 100 check_valid_data_fraction(job) @@ -2200,18 +2189,71 @@ def test_valid_filter(caplog, sc_3a_3b): assert "removing NIR016 for area euron1" in caplog.text assert "keeping IR037 for area euron1" in caplog.text assert "product absent not found, already removed" in caplog.text - tpg.reset_mock() + + +def test_valid_filter_small_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data with small coverage.""" + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): tpg.return_value = 1 - check_valid_data_fraction(job2) + check_valid_data_fraction(job) assert "inaccurate coverage estimate suspected!" in caplog.text - assert "NIR016" in prods2 - assert "IR037" in prods2 - tpg.reset_mock() + assert "NIR016" in prods + assert "IR037" in prods + + +def test_valid_filter_zero_coverage(caplog, sc_3a_3b): + """Test filter for minimum fraction of valid data without any coverage.""" + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): tpg.return_value = 0 - check_valid_data_fraction(job2) + check_valid_data_fraction(job) assert "no expected coverage at all, removing" in caplog.text - assert "NIR016" not in prods2 - assert "IR037" not in prods2 + assert "NIR016" not in prods + assert "IR037" not in prods + + +def test_valid_filter_no_trollsched(caplog, monkeypatch, sc_3a_3b): + """Test filter for minimum fraction of valid data with full coverage.""" + # This is needed when only this test is run + monkeypatch.setattr("trollsched.spherical.get_twilight_poly", None) + # ... and this when all the tests are run + monkeypatch.setattr("trollflow2.plugins.get_twilight_poly", None) + from trollflow2.plugins import check_valid_data_fraction + + job, prods = _create_valid_filter_job_and_prods(sc_3a_3b) + + with mock.patch("trollflow2.plugins._get_scene_coverage") as tpg, \ + caplog.at_level(logging.DEBUG): + tpg.return_value = 100 + check_valid_data_fraction(job) + + assert "Trollsched import failed" in caplog.text + assert "Keeping all products" in caplog.text + + +def _create_valid_filter_job_and_prods(sc_3a_3b): + from trollflow2.launcher import yaml + product_list = yaml.safe_load(yaml_test3) + job = {} + job['scene'] = sc_3a_3b + job['product_list'] = product_list.copy() + job['input_mda'] = input_mda.copy() + job['resampled_scenes'] = {"euron1": sc_3a_3b} + + prods = job['product_list']['product_list']['areas']['euron1']['products'] + for p in ("NIR016", "IR037", "absent"): + prods[p] = {"min_valid_data_fraction": 40} + + return job, prods def test_persisted(sc_3a_3b):