From 5904ba9016858061013c94264b9766c24afd7a18 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 28 Feb 2024 13:37:40 +0200 Subject: [PATCH 01/12] Update UTC handling --- trollflow2/plugins/__init__.py | 2 +- trollflow2/tests/test_trollflow2.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 9bd3b35a..fb32a071 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -686,7 +686,7 @@ def check_metadata(job): key) continue if key == 'start_time': - time_diff = dt.datetime.utcnow() - mda[key] + time_diff = dt.datetime.now(dt.UTC) - mda[key] if time_diff > abs(dt.timedelta(minutes=val)): age = "older" if val < 0 else "newer" raise AbortProcessing( diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 2e27338e..0fa1f651 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -284,7 +284,7 @@ !!python/object:trollflow2.plugins.FilePublisher {port: 40002, nameservers: [localhost]} """ -SCENE_START_TIME = dt.datetime.utcnow() +SCENE_START_TIME = dt.datetime.now(dt.UTC) SCENE_END_TIME = SCENE_START_TIME + dt.timedelta(minutes=15) JOB_INPUT_MDA_START_TIME = SCENE_START_TIME + dt.timedelta(seconds=10) @@ -1563,7 +1563,8 @@ def test_discard_new_data(self): """Test that new data are discarded.""" from trollflow2.plugins import AbortProcessing, check_metadata with mock.patch('trollflow2.plugins.get_config_value') as get_config_value: - job = {'product_list': None, 'input_mda': {'start_time': dt.datetime.utcnow() - dt.timedelta(minutes=90)}} + job = {'product_list': None, + 'input_mda': {'start_time': dt.datetime.now(dt.UTC) - dt.timedelta(minutes=90)}} get_config_value.return_value = {'start_time': +60} with self.assertRaises(AbortProcessing): check_metadata(job) From 567c95edd8458aaf681aae05e4856f1fa9f60a4c Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 28 Feb 2024 13:44:35 +0200 Subject: [PATCH 02/12] Remove usage of deprecated dpath.util --- continuous_integration/environment.yaml | 2 +- setup.py | 2 +- trollflow2/dict_tools.py | 6 +++--- trollflow2/plugins/__init__.py | 16 +++++++--------- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index ac41b0bf..b8e480c4 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -6,7 +6,7 @@ dependencies: - pytest - pytest-cov - pyyaml - - dpath + - dpath>=2.1.0 - trollsift - numpy - satpy>=0.32.0 diff --git a/setup.py b/setup.py index 688c3aa3..13f6af20 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ pass -install_requires = ['pyyaml', 'dpath', 'trollsift', 'posttroll>=1.10.0'] +install_requires = ['pyyaml', 'dpath>=2.1.0', 'trollsift', 'posttroll>=1.10.0'] if "test" not in sys.argv: install_requires += ['satpy>=0.32.0', 'pyorbital'] diff --git a/trollflow2/dict_tools.py b/trollflow2/dict_tools.py index 1e89940a..c1f83c80 100644 --- a/trollflow2/dict_tools.py +++ b/trollflow2/dict_tools.py @@ -20,7 +20,7 @@ # are not necessary """Tools for product list operations.""" -import dpath.util +import dpath def plist_iter(product_list, base_mda=None, level=None): @@ -88,11 +88,11 @@ def get_config_value(config, path, key, default=None): num = len(path_parts) for i in range(num, 1, -1): pwd = "/".join(path_parts[:i] + [key]) - vals = dpath.util.values(config, pwd) + vals = dpath.values(config, pwd) if len(vals) > 0: return vals[0] - vals = dpath.util.values(config, "/common/" + key) + vals = dpath.values(config, "/common/" + key) if len(vals) > 0: return vals[0] diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index fb32a071..51b4a174 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -33,7 +33,7 @@ import dask import dask.array as da -import dpath.util +import dpath import rasterio from dask.delayed import Delayed from posttroll.message import Message @@ -92,8 +92,6 @@ def create_scene(job): def load_composites(job): """Load composites given in the job's product_list.""" - # composites = set().union(*(set(d.keys()) - # for d in dpath.util.values(job['product_list'], '/product_list/areas/*/products'))) composites_by_res = {} for flat_prod_cfg, _prod_cfg in plist_iter(job['product_list']['product_list'], level='product'): res = flat_prod_cfg.get('resolution', DEFAULT) @@ -649,7 +647,7 @@ def _check_overall_coverage_for_area( "Area coverage %.2f %% below threshold %.2f %%", cov, min_coverage) logger.info("Removing area %s from the worklist", area) - dpath.util.delete(product_list, area_path) + dpath.delete(product_list, area_path) else: logger.debug(f"Area coverage {cov:.2f}% above threshold " @@ -748,7 +746,7 @@ def sza_check(job): if sunzen < limit: logger.info("Sun zenith angle too small for nighttime " "product '%s', product removed.", product) - dpath.util.delete(product_list, prod_path) + dpath.delete(product_list, prod_path) continue # Check daytime limit @@ -758,12 +756,12 @@ def sza_check(job): if sunzen > limit: logger.info("Sun zenith angle too large for daytime " "product '%s', product removed.", product) - dpath.util.delete(product_list, prod_path) + dpath.delete(product_list, prod_path) continue if len(product_list['product_list']['areas'][area]['products']) == 0: logger.info("Removing empty area: %s", area) - dpath.util.delete(product_list, '/product_list/areas/%s' % area) + dpath.delete(product_list, '/product_list/areas/%s' % area) def check_sunlight_coverage(job): @@ -843,12 +841,12 @@ def check_sunlight_coverage(job): logger.info("Not enough sunlight coverage for " f"product '{product!s}', removed. Needs at least " f"{min_day:.1f}%, got {coverage[check_pass]:.1%}.") - dpath.util.delete(product_list, prod_path) + dpath.delete(product_list, prod_path) if max_day is not None and coverage[check_pass] > (max_day / 100.0): logger.info("Too much sunlight coverage for " f"product '{product!s}', removed. Needs at most " f"{max_day:.1f}%, got {coverage[check_pass]:.1%}.") - dpath.util.delete(product_list, prod_path) + dpath.delete(product_list, prod_path) def _get_sunlight_coverage(area_def, start_time, overpass=None): From 0ba5d2d29a4135e9910ec7801ff2f8d10d8102f3 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 28 Feb 2024 13:54:52 +0200 Subject: [PATCH 03/12] Update area definition boundary computations --- trollflow2/plugins/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 51b4a174..52948559 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -40,7 +40,7 @@ from posttroll.publisher import create_publisher_from_dict_config from pyorbital.astronomy import sun_zenith_angle from pyresample.area_config import AreaNotFound -from pyresample.boundary import AreaDefBoundary, Boundary +from pyresample.boundary import Boundary from pyresample.geometry import get_geostationary_bounding_box from rasterio.enums import Resampling from satpy import Scene @@ -856,7 +856,7 @@ def _get_sunlight_coverage(area_def, start_time, overpass=None): *get_geostationary_bounding_box(area_def, nb_points=100)).contour_poly else: - adp = AreaDefBoundary(area_def, frequency=100).contour_poly + adp = area_def.boundary(vertices_per_side=100).contour_poly poly = get_twilight_poly(start_time) if overpass is not None: ovp = overpass.boundary.contour_poly From f2dd8cc7445da179e5a508e50a56188b9ff4893b Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 28 Feb 2024 14:05:12 +0200 Subject: [PATCH 04/12] Add tzinfo to datetime of input metadata --- trollflow2/tests/test_trollflow2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 0fa1f651..e9e42790 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1551,7 +1551,7 @@ def test_discard_old_data(self): from trollflow2.plugins import AbortProcessing, check_metadata with mock.patch('trollflow2.plugins.get_config_value') as get_config_value: get_config_value.return_value = None - job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18)}} + job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18, tzinfo=dt.UTC)}} self.assertIsNone(check_metadata(job)) get_config_value.return_value = {'start_time': -2e6} self.assertIsNone(check_metadata(job)) From 1214459c3faf02a379e0804914f65adaf2e2f970 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 29 Feb 2024 12:39:41 +0200 Subject: [PATCH 05/12] Fix coverage test to use adef.boundary --- trollflow2/tests/test_trollflow2.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index e9e42790..c4e63f38 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1109,17 +1109,16 @@ def setUp(self): def test_coverage(self): """Test sunlight coverage.""" from trollflow2.plugins import _get_sunlight_coverage - with mock.patch('trollflow2.plugins.AreaDefBoundary') as area_def_boundary, \ - mock.patch('trollflow2.plugins.Boundary') as boundary, \ + with mock.patch('trollflow2.plugins.Boundary') as boundary, \ mock.patch('trollflow2.plugins.get_twilight_poly'), \ mock.patch('trollflow2.plugins.get_area_def'), \ mock.patch('trollflow2.plugins.get_geostationary_bounding_box'): - area_def_boundary.return_value.contour_poly.intersection.return_value.area.return_value = 0.02 boundary.return_value.contour_poly.intersection.return_value.area.return_value = 0.02 - area_def_boundary.return_value.contour_poly.area.return_value = 0.2 - start_time = dt.datetime(2019, 4, 7, 20, 8) adef = mock.MagicMock(proj_dict={'proj': 'stere'}) + adef.boundary.return_value.contour_poly.intersection.return_value.area.return_value = 0.02 + adef.boundary.return_value.contour_poly.area.return_value = 0.2 + start_time = dt.datetime(2019, 4, 7, 20, 8) res = _get_sunlight_coverage(adef, start_time) np.testing.assert_allclose(res, 0.1) boundary.assert_not_called() From 54c73bb71fe23aac7acdd191bed7e13c910a8d3a Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 29 Feb 2024 12:48:41 +0200 Subject: [PATCH 06/12] Remove duplicate test --- trollflow2/tests/test_trollflow2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index c4e63f38..cc533c90 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1551,9 +1551,7 @@ def test_discard_old_data(self): with mock.patch('trollflow2.plugins.get_config_value') as get_config_value: get_config_value.return_value = None job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18, tzinfo=dt.UTC)}} - self.assertIsNone(check_metadata(job)) - get_config_value.return_value = {'start_time': -2e6} - self.assertIsNone(check_metadata(job)) + assert check_metadata(job) is None get_config_value.return_value = {'start_time': -60} with self.assertRaises(AbortProcessing): check_metadata(job) From 6373dee53ec77b6c8d32a26a0a6009c5feb81f0b Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 29 Feb 2024 12:58:08 +0200 Subject: [PATCH 07/12] Use dt.datetime.utc, dt.UTC isn't available in 3.10 --- trollflow2/plugins/__init__.py | 2 +- trollflow2/tests/test_trollflow2.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 52948559..11746bf9 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -684,7 +684,7 @@ def check_metadata(job): key) continue if key == 'start_time': - time_diff = dt.datetime.now(dt.UTC) - mda[key] + time_diff = dt.datetime.now(dt.timezone.utc) - mda[key] if time_diff > abs(dt.timedelta(minutes=val)): age = "older" if val < 0 else "newer" raise AbortProcessing( diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index cc533c90..0141c876 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -284,7 +284,7 @@ !!python/object:trollflow2.plugins.FilePublisher {port: 40002, nameservers: [localhost]} """ -SCENE_START_TIME = dt.datetime.now(dt.UTC) +SCENE_START_TIME = dt.datetime.now(dt.timezone.utc) SCENE_END_TIME = SCENE_START_TIME + dt.timedelta(minutes=15) JOB_INPUT_MDA_START_TIME = SCENE_START_TIME + dt.timedelta(seconds=10) @@ -1550,7 +1550,7 @@ def test_discard_old_data(self): from trollflow2.plugins import AbortProcessing, check_metadata with mock.patch('trollflow2.plugins.get_config_value') as get_config_value: get_config_value.return_value = None - job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18, tzinfo=dt.UTC)}} + job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18, tzinfo=dt.timezone.utc)}} assert check_metadata(job) is None get_config_value.return_value = {'start_time': -60} with self.assertRaises(AbortProcessing): @@ -1561,7 +1561,7 @@ def test_discard_new_data(self): from trollflow2.plugins import AbortProcessing, check_metadata with mock.patch('trollflow2.plugins.get_config_value') as get_config_value: job = {'product_list': None, - 'input_mda': {'start_time': dt.datetime.now(dt.UTC) - dt.timedelta(minutes=90)}} + 'input_mda': {'start_time': dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=90)}} get_config_value.return_value = {'start_time': +60} with self.assertRaises(AbortProcessing): check_metadata(job) From cdb62b6ce6b181352c0a2fac01299af1b6ea54d4 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 29 Feb 2024 13:25:08 +0200 Subject: [PATCH 08/12] Replace pkg_resources with importlib --- trollflow2/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/trollflow2/__init__.py b/trollflow2/__init__.py index 777ce6ba..f12076b1 100644 --- a/trollflow2/__init__.py +++ b/trollflow2/__init__.py @@ -25,14 +25,9 @@ # are not necessary """Base module for trollflow2.""" +from importlib.metadata import version from multiprocessing import Manager -from pkg_resources import DistributionNotFound, get_distribution - MP_MANAGER = Manager() -try: - __version__ = get_distribution(__name__).version -except DistributionNotFound: - # package is not installed - pass +__version__ = version(__name__) From 06f6a45840d0fdfa8ea6eca5b419aa346ed4e411 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 29 Feb 2024 13:30:34 +0200 Subject: [PATCH 09/12] Revert deletion of a test on discarding old data --- trollflow2/tests/test_trollflow2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 0141c876..5cfd43d3 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1552,6 +1552,8 @@ def test_discard_old_data(self): get_config_value.return_value = None job = {'product_list': None, 'input_mda': {'start_time': dt.datetime(2020, 3, 18, tzinfo=dt.timezone.utc)}} assert check_metadata(job) is None + get_config_value.return_value = {'start_time': -20e6} + assert check_metadata(job) is None get_config_value.return_value = {'start_time': -60} with self.assertRaises(AbortProcessing): check_metadata(job) From fe032c3500026bac202b106dfca7b3a6b98fe1c5 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 19 Apr 2024 14:04:35 +0300 Subject: [PATCH 10/12] Suppress warnings when reading non-georeferenced data --- trollflow2/tests/test_trollflow2.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 5cfd43d3..6a6c5b6b 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -711,14 +711,18 @@ def test_save_datasets_callback(tmp_path, caplog, fake_scene): def testlog(obj, targs, job, fmat_config): """Toy function doing some logging.""" + import warnings + filename = fmat_config["filename"] # ensure computation has indeed completed and file was flushed p = pathlib.Path(filename) logger.info(f"Wrote {filename} successfully, {p.stat().st_size:d} bytes") assert p.exists() - with rasterio.open(filename) as src: - arr = src.read(1) - assert arr[5, 5] == 142 + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="Dataset has no geotransform") + with rasterio.open(filename) as src: + arr = src.read(1) + assert arr[5, 5] == 142 return obj form = [ From 8b3bf3a9e2ffdc9cfdeea64b0a039ba4db77485e Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 19 Apr 2024 15:08:11 +0300 Subject: [PATCH 11/12] Do not convert area definition to dictionary to check for geos projection --- trollflow2/plugins/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trollflow2/plugins/__init__.py b/trollflow2/plugins/__init__.py index 11746bf9..6d89c28e 100644 --- a/trollflow2/plugins/__init__.py +++ b/trollflow2/plugins/__init__.py @@ -851,7 +851,7 @@ def check_sunlight_coverage(job): def _get_sunlight_coverage(area_def, start_time, overpass=None): """Get the sunlight coverage of *area_def* at *start_time* as a value between 0 and 1.""" - if area_def.proj_dict.get('proj') == 'geos': + if area_def.is_geostationary: adp = Boundary( *get_geostationary_bounding_box(area_def, nb_points=100)).contour_poly From 643b5c7e918beb479b373498910ea6253ad23723 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Fri, 19 Apr 2024 15:12:47 +0300 Subject: [PATCH 12/12] Replace proj dict mocks with is_geostationary attribute --- trollflow2/tests/test_trollflow2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trollflow2/tests/test_trollflow2.py b/trollflow2/tests/test_trollflow2.py index 6a6c5b6b..30352e40 100644 --- a/trollflow2/tests/test_trollflow2.py +++ b/trollflow2/tests/test_trollflow2.py @@ -1119,14 +1119,14 @@ def test_coverage(self): mock.patch('trollflow2.plugins.get_geostationary_bounding_box'): boundary.return_value.contour_poly.intersection.return_value.area.return_value = 0.02 - adef = mock.MagicMock(proj_dict={'proj': 'stere'}) + adef = mock.MagicMock(is_geostationary=False) adef.boundary.return_value.contour_poly.intersection.return_value.area.return_value = 0.02 adef.boundary.return_value.contour_poly.area.return_value = 0.2 start_time = dt.datetime(2019, 4, 7, 20, 8) res = _get_sunlight_coverage(adef, start_time) np.testing.assert_allclose(res, 0.1) boundary.assert_not_called() - adef = mock.MagicMock(proj_dict={'proj': 'geos'}) + adef = mock.MagicMock(is_geostationary=True) res = _get_sunlight_coverage(adef, start_time) boundary.assert_called()