From 1cd358189477b5f0d47bf51c54eaa08c5c47e749 Mon Sep 17 00:00:00 2001 From: BENR0 Date: Wed, 28 Dec 2022 12:42:35 +0100 Subject: [PATCH 1/7] refactor: add compat keyword to to_xarray_dataset --- satpy/scene.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/satpy/scene.py b/satpy/scene.py index 92e9ff5615..cff5648849 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1036,12 +1036,16 @@ def to_geoviews(self, gvtype=None, datasets=None, kdims=None, vdims=None, dynami return gview - def to_xarray_dataset(self, datasets=None): + def to_xarray_dataset(self, datasets=None, compat="minimal"): """Merge all xr.DataArrays of a scene to a xr.DataSet. Parameters: datasets (list): List of products to include in the :class:`xarray.Dataset` + compat (Optional[str]): + How to compare variables with the same name for conflicts. + See :func:`xarray.merge` for possible options. Defaults to + "minimal" which drops conflicting variables. Returns: :class:`xarray.Dataset` @@ -1055,7 +1059,7 @@ def to_xarray_dataset(self, datasets=None): mdata = combine_metadata(*tuple(i.attrs for i in dataarrays)) if mdata.get('area') is None or not isinstance(mdata['area'], SwathDefinition): # either don't know what the area is or we have an AreaDefinition - ds = xr.merge(ds_dict.values()) + ds = xr.merge(ds_dict.values(), compat=compat) else: # we have a swath definition and should use lon/lat values lons, lats = mdata['area'].get_lonlats() From 549051527a065eeeb8b6376c59ff2be4b16fdd41 Mon Sep 17 00:00:00 2001 From: BENR0 Date: Wed, 28 Dec 2022 12:51:13 +0100 Subject: [PATCH 2/7] tests: add tests --- satpy/tests/test_scene.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/satpy/tests/test_scene.py b/satpy/tests/test_scene.py index 1d9386d014..0050a783d3 100644 --- a/satpy/tests/test_scene.py +++ b/satpy/tests/test_scene.py @@ -2036,6 +2036,36 @@ def test_to_xarray_dataset_with_empty_scene(self): assert len(xrds.variables) == 0 assert len(xrds.coords) == 0 + def test_to_xarray_dataset_with_conflicting_variables(self): + """Test converting Scene with DataArrays with conflicting variables. + + E.g. "acq_time" in the seviri_l1b_nc reader + """ + scn = Scene() + + acq_time_1 = ('y', [np.datetime64('1958-01-02 00:00:01'), + np.datetime64('1958-01-02 00:00:02')]) + ds = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=('y', 'x'), + attrs={'start_time': datetime(2018, 1, 1)}) + ds['acq_time'] = acq_time_1 + + scn['ds1'] = ds + + acq_time_2 = ('y', [np.datetime64('1958-02-02 00:00:01'), + np.datetime64('1958-02-02 00:00:02')]) + ds['acq_time'] = acq_time_2 + + scn['ds2'] = ds + + xrds = scn.to_xarray_dataset() + assert isinstance(xrds, xr.Dataset) + assert 'acq_time' not in xrds.coords + + xrds = scn.to_xarray_dataset(compat='override') + assert isinstance(xrds, xr.Dataset) + assert 'acq_time' in xrds.coords + assert xrds['acq_time'] == acq_time_1 + def test_geoviews_basic_with_area(self): """Test converting a Scene to geoviews with an AreaDefinition.""" from pyresample.geometry import AreaDefinition From 904270cb72045552fa877b7ac5f1b24379ecbc28 Mon Sep 17 00:00:00 2001 From: BENR0 Date: Fri, 14 Jun 2024 14:05:01 +0200 Subject: [PATCH 3/7] fix: compat=override test --- satpy/tests/scene_tests/test_conversions.py | 46 +++++++++++++++++---- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/satpy/tests/scene_tests/test_conversions.py b/satpy/tests/scene_tests/test_conversions.py index 4490903880..50ea1a46d8 100644 --- a/satpy/tests/scene_tests/test_conversions.py +++ b/satpy/tests/scene_tests/test_conversions.py @@ -17,7 +17,9 @@ """Unit tests for Scene conversion functionality.""" import datetime as dt +from datetime import datetime +import numpy as np import pytest import xarray as xr from dask import array as da @@ -47,14 +49,6 @@ def test_serialization_with_readers_and_data_arr(self): class TestSceneConversions: """Test Scene conversion to geoviews, xarray, etc.""" - def test_to_xarray_dataset_with_empty_scene(self): - """Test converting empty Scene to xarray dataset.""" - scn = Scene() - xrds = scn.to_xarray_dataset() - assert isinstance(xrds, xr.Dataset) - assert len(xrds.variables) == 0 - assert len(xrds.coords) == 0 - def test_geoviews_basic_with_area(self): """Test converting a Scene to geoviews with an AreaDefinition.""" from pyresample.geometry import AreaDefinition @@ -157,6 +151,42 @@ def single_area_scn(self): scn["var1"] = data_array return scn + def test_to_xarray_dataset_with_conflicting_variables(self): + """Test converting Scene with DataArrays with conflicting variables. + + E.g. "acq_time" in the seviri_l1b_nc reader + """ + from pyresample.geometry import AreaDefinition + area = AreaDefinition("test", "test", "test", + {"proj": "geos", "lon_0": -95.5, "h": 35786023.0}, + 2, 2, [-200, -200, 200, 200]) + scn = Scene() + + acq_time_1 = ("y", [np.datetime64("1958-01-02 00:00:01"), + np.datetime64("1958-01-02 00:00:02")]) + ds = xr.DataArray(da.zeros((2, 2), chunks=-1), dims=("y", "x"), + attrs={"start_time": datetime(2018, 1, 1), "area": area}) + ds["acq_time"] = acq_time_1 + + scn["ds1"] = ds + + acq_time_2 = ("y", [np.datetime64("1958-02-02 00:00:01"), + np.datetime64("1958-02-02 00:00:02")]) + ds2 = ds.copy() + ds2["acq_time"] = acq_time_2 + + scn["ds2"] = ds2 + + # drop case (compat="minimal") + xrds = scn.to_xarray_dataset() + assert isinstance(xrds, xr.Dataset) + assert "acq_time" not in xrds.coords + + xrds = scn.to_xarray_dataset(compat="override") + assert isinstance(xrds, xr.Dataset) + assert "acq_time" in xrds.coords + np.testing.assert_array_equal(xrds["acq_time"].values, acq_time_1[1]) + @pytest.fixture() def multi_area_scn(self): """Define Scene with multiple area.""" From aff7b96c114fe0d9e1283b03d7afa987cc3b08d5 Mon Sep 17 00:00:00 2001 From: BENR0 Date: Fri, 14 Jun 2024 14:47:55 +0200 Subject: [PATCH 4/7] fix: testing of datasets euqal --- satpy/tests/scene_tests/test_conversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/scene_tests/test_conversions.py b/satpy/tests/scene_tests/test_conversions.py index 50ea1a46d8..f8d4c0a0cb 100644 --- a/satpy/tests/scene_tests/test_conversions.py +++ b/satpy/tests/scene_tests/test_conversions.py @@ -185,7 +185,7 @@ def test_to_xarray_dataset_with_conflicting_variables(self): xrds = scn.to_xarray_dataset(compat="override") assert isinstance(xrds, xr.Dataset) assert "acq_time" in xrds.coords - np.testing.assert_array_equal(xrds["acq_time"].values, acq_time_1[1]) + xr.testing.assert_equal(xrds["acq_time"], ds["acq_time"]) @pytest.fixture() def multi_area_scn(self): From d8905baf48244b5ace4ac3a787f41bd6a9a831d7 Mon Sep 17 00:00:00 2001 From: BENR0 Date: Wed, 17 Jul 2024 12:26:21 +0200 Subject: [PATCH 5/7] refactor: remove Optional type string --- satpy/scene.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/scene.py b/satpy/scene.py index d2a8b0d002..03bf09ce56 100644 --- a/satpy/scene.py +++ b/satpy/scene.py @@ -1082,7 +1082,7 @@ def to_xarray_dataset(self, datasets=None, compat="minimal"): Parameters: datasets (list): List of products to include in the :class:`xarray.Dataset` - compat (Optional[str]): + compat (str): How to compare variables with the same name for conflicts. See :func:`xarray.merge` for possible options. Defaults to "minimal" which drops conflicting variables. From 3da840dc6e9a8366bd344dc9d97009d5d229435b Mon Sep 17 00:00:00 2001 From: BENR0 Date: Wed, 17 Jul 2024 13:07:03 +0200 Subject: [PATCH 6/7] fix: order of dataarrays --- satpy/tests/scene_tests/test_conversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/tests/scene_tests/test_conversions.py b/satpy/tests/scene_tests/test_conversions.py index f8d4c0a0cb..2ee8fb8f73 100644 --- a/satpy/tests/scene_tests/test_conversions.py +++ b/satpy/tests/scene_tests/test_conversions.py @@ -185,7 +185,7 @@ def test_to_xarray_dataset_with_conflicting_variables(self): xrds = scn.to_xarray_dataset(compat="override") assert isinstance(xrds, xr.Dataset) assert "acq_time" in xrds.coords - xr.testing.assert_equal(xrds["acq_time"], ds["acq_time"]) + xr.testing.assert_equal(xrds["acq_time"], ds2["acq_time"]) @pytest.fixture() def multi_area_scn(self): From b465b0a1c8f86e5d52798bc1487c25a12a9fd249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20R=C3=B6sner?= Date: Tue, 19 Nov 2024 10:46:52 +0100 Subject: [PATCH 7/7] fix: test --- satpy/tests/scene_tests/test_conversions.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/satpy/tests/scene_tests/test_conversions.py b/satpy/tests/scene_tests/test_conversions.py index 2e7d1010fa..96a19b7ed7 100644 --- a/satpy/tests/scene_tests/test_conversions.py +++ b/satpy/tests/scene_tests/test_conversions.py @@ -189,10 +189,11 @@ def test_to_xarray_dataset_with_conflicting_variables(self): assert isinstance(xrds, xr.Dataset) assert "acq_time" not in xrds.coords - xrds = scn.to_xarray_dataset(compat="override") + # override: pick variable from first dataset + xrds = scn.to_xarray_dataset(datasets=["ds1", "ds2"], compat="override") assert isinstance(xrds, xr.Dataset) assert "acq_time" in xrds.coords - xr.testing.assert_equal(xrds["acq_time"], ds2["acq_time"]) + xr.testing.assert_equal(xrds["acq_time"], ds["acq_time"]) @pytest.fixture def multi_area_scn(self):