Merge branch 'download_improvements_retry'

src/coastsat/SDS_download.py

@@ -14,7 +14,6 @@
 from typing import List, Dict, Union, Tuple
 import time
 from functools import wraps
-import traceback
 
 # earth engine module
 import ee
@@ -29,7 +28,6 @@
 # additional modules
 from datetime import datetime, timedelta
 import pytz
-import pickle
 from skimage import morphology, transform
 from scipy import ndimage
 
@@ -121,7 +119,7 @@ def retry(func):
     def wrapper(*args, **kwargs):
         max_tries = 3
         image_id = kwargs.get(
-            "image_id", "Unknown"
+            "image_id", "Unknown image id"
         )  # Get image_id from kwargs or use 'Unknown'
         for i in range(max_tries):
             try:
@@ -145,6 +143,20 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
+@retry
+def remove_dimensions_from_bands(image_ee):
+    # first delete dimensions key from dictionary
+    # otherwise the entire image is extracted (don't know why)
+    try:
+        im_bands = image_ee.getInfo()["bands"]
+        for j in range(len(im_bands)):
+            del im_bands[j]["dimensions"]
+        return im_bands
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        raise
+
+
 def get_image_quality(satname: str, im_meta: dict) -> str:
     """
     Get the image quality for a given satellite name and image metadata.
@@ -450,9 +462,7 @@ def retrieve_images(
 
                 # first delete dimensions key from dictionary
                 # otherwise the entire image is extracted (don't know why)
-                im_bands = image_ee.getInfo()["bands"]
-                for j in range(len(im_bands)):
-                    del im_bands[j]["dimensions"]
+                im_bands = remove_dimensions_from_bands(image_ee)
 
                 # =============================================================================================#
                 # Landsat 5 download
@@ -1171,9 +1181,10 @@ def get_s2cloudless(image_list: list, inputs: dict):
     return matched_cloud_images
 
 
+@retry  # Apply the retry decorator to the function
 def get_image_info(collection, satname, polygon, dates, **kwargs):
     """
-    Reads info about EE images for the specified collection, satellite and dates
+    Reads info about EE images for the specified collection, satellite, and dates
 
     KV WRL 2022
 
@@ -1193,25 +1204,18 @@ def get_image_info(collection, satname, polygon, dates, **kwargs):
     im_list: list of ee.Image objects
         list with the info for the images
     """
-    while True:
-        try:
-            # get info about images
-            ee_col = ee.ImageCollection(collection)
-            # Initialize the collection with filterBounds and filterDate
-            col = ee_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(
-                dates[0], dates[1]
-            )
-            # If "S2tile" key is in kwargs and its associated value is truthy (not an empty string, None, etc.),
-            # then apply an additional filter to the collection.
-            if kwargs.get("S2tile"):
-                col = col.filterMetadata(
-                    "MGRS_TILE", "equals", kwargs["S2tile"]
-                )  # 58GGP
-                print(f"Only keeping user-defined S2tile: {kwargs['S2tile']}")
-            im_list = col.getInfo().get("features")
-            break
-        except:
-            continue
+    # get info about images
+    ee_col = ee.ImageCollection(collection)
+    # Initialize the collection with filterBounds and filterDate
+    col = ee_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate(
+        dates[0], dates[1]
+    )
+    # If "S2tile" key is in kwargs and its associated value is truthy (not an empty string, None, etc.),
+    # then apply an additional filter to the collection.
+    if kwargs.get("S2tile"):
+        col = col.filterMetadata("MGRS_TILE", "equals", kwargs["S2tile"])  # 58GGP
+        print(f"Only keeping user-defined S2tile: {kwargs['S2tile']}")
+    im_list = col.getInfo().get("features")
     # remove very cloudy images (>95% cloud cover)
     im_list = remove_cloudy_images(im_list, satname)
     return im_list

test_check_images_available.py

@@ -0,0 +1,79 @@
+# ==========================================================#
+# Shoreline extraction from satellite images
+# ==========================================================#
+
+# Kilian Vos WRL 2018
+
+# %% 1. Initial settings
+
+# load modules
+import os
+import numpy as np
+import pickle
+import warnings
+
+warnings.filterwarnings("ignore")
+import matplotlib.pyplot as plt
+from matplotlib import gridspec
+
+plt.ion()
+import pandas as pd
+from scipy import interpolate
+from scipy import stats
+from datetime import datetime, timedelta
+import pytz
+from coastsat import (
+    SDS_download,
+    SDS_preprocess,
+    SDS_shoreline,
+    SDS_tools,
+    SDS_transects,
+)
+
+# region of interest (longitude, latitude in WGS84)
+polygon = [
+    [
+        [-117.51525453851755, 33.29929946257779],
+        [-117.51549193684221, 33.33963611101142],
+        [-117.46743525920786, 33.33982627488089],
+        [-117.46721999025168, 33.2994893366537],
+        [-117.51525453851755, 33.29929946257779],
+    ]
+]
+# can also be loaded from a .kml polygon
+# kml_polygon = os.path.join(os.getcwd(), 'examples', 'NARRA_polygon.kml')
+# polygon = SDS_tools.polygon_from_kml(kml_polygon)
+# convert polygon to a smallest rectangle (sides parallel to coordinate axes)
+# polygon = SDS_tools.smallest_rectangle(polygon)
+
+# date range
+# dates = ["1984-01-01", "2022-01-01"]
+# dates = ["2022-01-01", "2022-12-01"]
+# dates = ["2014-06-01", "2015-03-01"]
+dates = ["2014-04-29", "2014-06-01"]
+
+
+# satellite missions
+# sat_list = ["L5", "L7", "L8"]
+# sat_list = ["L5", "L7", "L8", "L9", "S2"]
+sat_list = [
+    "L8",
+]
+collection = "C02"  # choose Landsat collection 'C01' or 'C02'
+# name of the site
+sitename = "l8_failure2"
+# filepath where data will be stored
+filepath_data = os.path.join(os.getcwd(), "data")
+
+# put all the inputs into a dictionnary
+inputs = {
+    "polygon": polygon,
+    "dates": dates,
+    "sat_list": sat_list,
+    "sitename": sitename,
+    "filepath": filepath_data,
+    "landsat_collection": collection,
+    "include_T2": True,
+}
+
+SDS_download.check_images_available(inputs)