Fix management of "closestToDate" in GEE modules

geodataflow/pipeline/filters/GEEProductCatalog.py

@@ -31,9 +31,10 @@
 ===============================================================================
 """
 
+import datetime
 import importlib
 import requests
-from typing import Dict, List
+from typing import Dict, List, Iterable
 from shapely.geometry import mapping as shapely_mapping, shape as shapely_shape
 from shapely.geometry.polygon import Polygon
 from geodataflow.core.common import DateUtils
@@ -233,7 +234,7 @@ def ee_iter_function(ee_image, new_list):
             ee_dataset = self.ee_fetch_dataset(geometry, False)
             ee_images = ee_dataset.iterate(ee_iter_function, ee.List([]))
 
-            for image_props in ee_images.getInfo():
+            for image_props in self.pass_images(ee_images.getInfo()):
                 properties = feature.properties.copy()
 
                 properties['productType'] = self.dataset
@@ -264,6 +265,31 @@ def ee_iter_function(ee_image, new_list):
 
         pass
 
+    def pass_images(self, images: Iterable[object]) -> Iterable[object]:
+        """
+        Returns only those GEE Images that match custom criteria (e.g. "closestToDate").
+        """
+        if self.closestToDate:
+            closest_time = datetime.datetime.strptime(self.closestToDate, '%Y-%m-%d')
+            temp_dict = dict()
+            temp_list = list()
+
+            for image in images:
+                product_date = image.get('IMAGE_DATE')
+                product_time = datetime.datetime.strptime(product_date, '%Y-%m-%d')
+                product_diff = abs(product_time - closest_time).days
+                temp_list.append((product_date, product_time, product_diff))
+                curr_list = temp_dict.get(product_date, [])
+                curr_list.append(image)
+                temp_dict[product_date] = curr_list
+
+            for best_item in sorted(temp_list, key=lambda tup: tup[2], reverse=False):
+                best_date = best_item[0]
+                best_list = temp_dict[best_date]
+                return best_list
+
+        return images
+
     def ee_initialize(self):
         """
         Returns a GEE environemt.

geodataflow/pipeline/filters/GEEProductDataset.py

@@ -221,13 +221,23 @@ def ee_enumerate_subsets(self, ee_dataset, ee_dataset_props, ee_image_props) ->
         Enumerate the GEE Image set of the ImageCollection.
         """
         ee = self.ee_initialize()
+
         dataset_props = ee_dataset_props.getInfo()
+        best_date = None
+
+        if self.closestToDate and dataset_props['count'] > 1:
+            temp_sets = [{'IMAGE_DATE': product_date} for product_date in dataset_props['dates']]
+            temp_sets = self.pass_images(temp_sets)
+            best_date = temp_sets[0].get('IMAGE_DATE')
 
         if self.groupByDate:
             for product_date in dataset_props['dates']:
                 y, m, d = \
                     int(product_date[0:4]), int(product_date[5:7]), int(product_date[8:10])
 
+                if best_date and product_date != best_date:
+                    continue
+
                 ee_subset = ee_dataset \
                     .filter(ee.Filter.calendarRange(y, y, 'year')) \
                     .filter(ee.Filter.calendarRange(m, m, 'month')) \
@@ -243,6 +253,11 @@ def ee_enumerate_subsets(self, ee_dataset, ee_dataset_props, ee_image_props) ->
             for i in range(0, count):
                 ee_image = ee.Image(ee_dataset.toList(count).get(i))
                 ee_product_date = ee.Date(ee_image.get('system:time_start')).format('yyyy-MM-dd')
-                yield ee_product_date.getInfo(), ee_image
+                product_date = ee_product_date.getInfo()
+
+                if best_date and product_date != best_date:
+                    continue
+
+                yield product_date, ee_image
 
         pass

tests/test_geodataflow.py

@@ -308,7 +308,7 @@ def test_gee_collection_catalog(self):
 
         def test_func(features):
             """ Test results """
-            self.assertEqual(len(features), 4)
+            self.assertEqual(len(features), 2)
             feature = features[0]
             self.assertEqual(feature.type, 'Feature')
             self.assertEqual(feature.geometry.geom_type, 'Polygon')
@@ -330,7 +330,7 @@ def test_gee_collection_dataset(self):
 
         def test_func(features):
             """ Test results """
-            self.assertEqual(len(features), 2)
+            self.assertEqual(len(features), 1)
             feature = features[0]
             self.assertEqual(feature.type, 'Raster')
             self.assertEqual(feature.geometry.geom_type, 'Polygon')