From 31e4cef1b4e19bcad802a44563d29a88495d61af Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 12:56:18 +0100 Subject: [PATCH 1/7] FIX: ignore bands not present in reference scene (Landsat issue) --- eodal/mapper/mapper.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/eodal/mapper/mapper.py b/eodal/mapper/mapper.py index 413739b..4d8d162 100644 --- a/eodal/mapper/mapper.py +++ b/eodal/mapper/mapper.py @@ -737,9 +737,20 @@ def _load_scenes_collection( for _, scene in scoll: # loop over the bands and reproject them # onto the target grid + bands_to_remove = [] for band_name, band in scene: - # get the reference band - reference_band = reference_scene[band_name] + # get the reference band. For some platforms, there + # might be different bands in the scene (different Landsat + # generations of sensors). In this case we have to continue. + try: + reference_band = reference_scene[band_name] + except KeyError: + # if band is not in reference scene, we memorize + # the band name and continue. We delete the band + # later to make sure the scenes in the collection + # all have the same number of bands + bands_to_remove.append(band_name) + continue geo_info = reference_band.geo_info # get the transform of the destination extent minx, maxy = total_bounds[0], total_bounds[-1] @@ -769,6 +780,12 @@ def _load_scenes_collection( dst_transform=dst_transform, destination=destination, dst_nodata=dst_nodata) + # remove bands if any + if len(bands_to_remove) > 0: + for band_to_remove in bands_to_remove: + del scene[band_to_remove] + if band_to_remove in scene.band_aliases: + scene.band_aliases.remove(band_to_remove) self.data = scoll From c9a32a04d72ab32eae54936ab2822d9a38242510 Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:01:16 +0100 Subject: [PATCH 2/7] changed signs in offset calculation to be consistent with QGIS --- eodal/core/band.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eodal/core/band.py b/eodal/core/band.py index ab81e11..4fc06bc 100644 --- a/eodal/core/band.py +++ b/eodal/core/band.py @@ -2230,24 +2230,24 @@ def scale_data( scale, offset = self.scale, self.offset if self.is_masked_array: if pixel_values_to_ignore is None: - scaled_array = scale * self.values.data - offset + scaled_array = scale * self.values.data + offset else: scaled_array = self.values.data.copy().astype(float) scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] = scale * \ scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] \ - - offset + + offset # reuse fill value fill_value = self.values.fill_value scaled_array = np.ma.MaskedArray( data=scaled_array, mask=self.values.mask, fill_value=fill_value) elif self.is_ndarray: if pixel_values_to_ignore is None: - scaled_array = scale * self.values - offset + scaled_array = scale * self.values + offset else: scaled_array = self.values.copy().astype(float) scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] = scale * \ scaled_array[~np.isin(scaled_array, pixel_values_to_ignore)] \ - - offset + + offset elif self.is_zarr: raise NotImplementedError() From 4b175c397a238b9fdd38c6d3fcf86d5ce2ca014d Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:01:36 +0100 Subject: [PATCH 3/7] changed sign for offset from `+0.1` to `-0.1` --- eodal/core/sensors/sentinel2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eodal/core/sensors/sentinel2.py b/eodal/core/sensors/sentinel2.py index 9206de0..98b4720 100644 --- a/eodal/core/sensors/sentinel2.py +++ b/eodal/core/sensors/sentinel2.py @@ -117,7 +117,7 @@ def _get_gain_and_offset(in_dir: Union[str, Path]) -> Tuple[Number, Number]: # actual reflectance factor values. s2_offset = 0 if baseline >= 400: - s2_offset = 0.1 + s2_offset = -0.1 return (s2_gain_factor, s2_offset) @staticmethod From 49f3621fef8cb5042e84bf007640bcaaf1d57980 Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:13:28 +0100 Subject: [PATCH 4/7] fixed failed test (changed sign of offset) --- tests/core/test_raster_collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/test_raster_collection.py b/tests/core/test_raster_collection.py index 060615e..a03afd6 100644 --- a/tests/core/test_raster_collection.py +++ b/tests/core/test_raster_collection.py @@ -136,7 +136,7 @@ def test_scale(): ones = np.ones((100,100), dtype='uint16') band_name_ones = 'ones' scale = 100 - offset = 2 + offset = -2 handler.add_band( Band, band_name=band_name_ones, @@ -150,7 +150,7 @@ def test_scale(): before_scaling = handler.get_values() handler.scale(inplace=True) after_scaling = handler.get_values() - assert (after_scaling == scale * before_scaling - offset).all(), 'values not scaled correctly' + assert (after_scaling == scale * before_scaling + offset).all(), 'values not scaled correctly' assert (after_scaling == 98).all(), 'wrong value after scaling' From ab9d0a5e468996cd574883d95685e3f900842962 Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:58:56 +0100 Subject: [PATCH 5/7] updated version `0.2.3` -> `0.2.4` --- eodal/__meta__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eodal/__meta__.py b/eodal/__meta__.py index 5ea3e41..cd5679b 100644 --- a/eodal/__meta__.py +++ b/eodal/__meta__.py @@ -15,4 +15,4 @@ description = "Earth Observation Data Analysis Library" # One-liner url = "https://github.com/EOA-team/eodal" # your project home-page license = "GNU General Public License version 3" # See https://choosealicense.com -version = "0.2.3" +version = "0.2.4" diff --git a/setup.py b/setup.py index 199bdd5..9288274 100755 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ def run(self): name='eodal', # setup_requires=['setuptools_scm'], # use_scm_version={'version_scheme': 'python-simplified-semver'}, - version='0.2.3', + version='0.2.4', description='The Earth Observation Data Analysis Library EOdal', long_description=long_description, long_description_content_type='text/markdown', From 84999c15256ccc3de8383c11b91cf029f0f326bb Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:59:09 +0100 Subject: [PATCH 6/7] updated `omit` section --- .coveragerc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index 7c54860..2a9c7e4 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,6 @@ [run] branch = True -omit = */tests/*, eodal/downloader/, eodal/metadata/database/ +omit = */tests/*, */downloader/*, */metadata/*/database/* [report] exclude_lines = From 805fe24c1703ea0f0a588c03e14272e8e5db9be7 Mon Sep 17 00:00:00 2001 From: lukas Date: Fri, 1 Dec 2023 13:59:20 +0100 Subject: [PATCH 7/7] updated changelog for version `0.2.4` --- CHANGELOG.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 37e682b..4182acb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,10 +10,19 @@ The format is based on `Keep a Changelog`_, and this project adheres to `Semanti Categories for changes are: Added, Changed, Deprecated, Removed, Fixed, Security. +Version `0.2.3 < https://github.com/EOA-team/eodal/releases/tag/v0.2.4>`__ +-------------------------------------------------------------------------------- + +Release date: 2023-12-01 + +- Fixed: selection of reference bands fixed for Landsat (different sensors have different quality bands). Now, only those quality bands available for all sensor generations are retained in case there are multiple different Landsat sensor generation involved. +- Fixed: sign of offset changed from - to + to be consistent with QGIS when scaling data. + + Version `0.2.3 < https://github.com/EOA-team/eodal/releases/tag/v0.2.3>`__ -------------------------------------------------------------------------------- -Release date: 2023-10-XX +Release date: 2023-11-24 - Fixed: a fill value is now explicitly set to each `np.ma.masked_array`. If not (previous behavior) numpy would interfere a fill value that would often not be the no-data type of the raster. In the forthcoming release of EOdal, the fill-value will now equal the nodata value. - Fixed: the inference of the highest data type in a `RasterCollection` has been improved and extended to all `INT`, `FLOAT`, and `COMPLEX` data types currently supported by numpy. In the previous version, all data types were cast to `numpy.float32` or even `numpy.float64` leading to an unnecessary high consumption of memory.