diff --git a/geodataflow/pipeline/filters/GEEProductCatalog.py b/geodataflow/pipeline/filters/GEEProductCatalog.py index 808475e..a107bf5 100644 --- a/geodataflow/pipeline/filters/GEEProductCatalog.py +++ b/geodataflow/pipeline/filters/GEEProductCatalog.py @@ -31,9 +31,10 @@ =============================================================================== """ +import datetime import importlib import requests -from typing import Dict, List +from typing import Dict, List, Iterable from shapely.geometry import mapping as shapely_mapping, shape as shapely_shape from shapely.geometry.polygon import Polygon from geodataflow.core.common import DateUtils @@ -233,7 +234,7 @@ def ee_iter_function(ee_image, new_list): ee_dataset = self.ee_fetch_dataset(geometry, False) ee_images = ee_dataset.iterate(ee_iter_function, ee.List([])) - for image_props in ee_images.getInfo(): + for image_props in self.pass_images(ee_images.getInfo()): properties = feature.properties.copy() properties['productType'] = self.dataset @@ -264,6 +265,31 @@ def ee_iter_function(ee_image, new_list): pass + def pass_images(self, images: Iterable[object]) -> Iterable[object]: + """ + Returns only those GEE Images that match custom criteria (e.g. "closestToDate"). + """ + if self.closestToDate: + closest_time = datetime.datetime.strptime(self.closestToDate, '%Y-%m-%d') + temp_dict = dict() + temp_list = list() + + for image in images: + product_date = image.get('IMAGE_DATE') + product_time = datetime.datetime.strptime(product_date, '%Y-%m-%d') + product_diff = abs(product_time - closest_time).days + temp_list.append((product_date, product_time, product_diff)) + curr_list = temp_dict.get(product_date, []) + curr_list.append(image) + temp_dict[product_date] = curr_list + + for best_item in sorted(temp_list, key=lambda tup: tup[2], reverse=False): + best_date = best_item[0] + best_list = temp_dict[best_date] + return best_list + + return images + def ee_initialize(self): """ Returns a GEE environemt. diff --git a/geodataflow/pipeline/filters/GEEProductDataset.py b/geodataflow/pipeline/filters/GEEProductDataset.py index 4a261ee..19a495b 100644 --- a/geodataflow/pipeline/filters/GEEProductDataset.py +++ b/geodataflow/pipeline/filters/GEEProductDataset.py @@ -221,13 +221,23 @@ def ee_enumerate_subsets(self, ee_dataset, ee_dataset_props, ee_image_props) -> Enumerate the GEE Image set of the ImageCollection. """ ee = self.ee_initialize() + dataset_props = ee_dataset_props.getInfo() + best_date = None + + if self.closestToDate and dataset_props['count'] > 1: + temp_sets = [{'IMAGE_DATE': product_date} for product_date in dataset_props['dates']] + temp_sets = self.pass_images(temp_sets) + best_date = temp_sets[0].get('IMAGE_DATE') if self.groupByDate: for product_date in dataset_props['dates']: y, m, d = \ int(product_date[0:4]), int(product_date[5:7]), int(product_date[8:10]) + if best_date and product_date != best_date: + continue + ee_subset = ee_dataset \ .filter(ee.Filter.calendarRange(y, y, 'year')) \ .filter(ee.Filter.calendarRange(m, m, 'month')) \ @@ -243,6 +253,11 @@ def ee_enumerate_subsets(self, ee_dataset, ee_dataset_props, ee_image_props) -> for i in range(0, count): ee_image = ee.Image(ee_dataset.toList(count).get(i)) ee_product_date = ee.Date(ee_image.get('system:time_start')).format('yyyy-MM-dd') - yield ee_product_date.getInfo(), ee_image + product_date = ee_product_date.getInfo() + + if best_date and product_date != best_date: + continue + + yield product_date, ee_image pass diff --git a/tests/test_geodataflow.py b/tests/test_geodataflow.py index 372b670..72a12d6 100644 --- a/tests/test_geodataflow.py +++ b/tests/test_geodataflow.py @@ -308,7 +308,7 @@ def test_gee_collection_catalog(self): def test_func(features): """ Test results """ - self.assertEqual(len(features), 4) + self.assertEqual(len(features), 2) feature = features[0] self.assertEqual(feature.type, 'Feature') self.assertEqual(feature.geometry.geom_type, 'Polygon') @@ -330,7 +330,7 @@ def test_gee_collection_dataset(self): def test_func(features): """ Test results """ - self.assertEqual(len(features), 2) + self.assertEqual(len(features), 1) feature = features[0] self.assertEqual(feature.type, 'Raster') self.assertEqual(feature.geometry.geom_type, 'Polygon')