From a08a294e1eea9c86e60f935603e1832acaa2729a Mon Sep 17 00:00:00 2001 From: Sharon Fitzpatrick Date: Thu, 9 Nov 2023 14:12:49 -0800 Subject: [PATCH 1/3] add retry im_bands = image_ee.getInfo()["bands"] --- src/coastsat/SDS_download.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/coastsat/SDS_download.py b/src/coastsat/SDS_download.py index f808ed8..f749806 100644 --- a/src/coastsat/SDS_download.py +++ b/src/coastsat/SDS_download.py @@ -145,6 +145,20 @@ def wrapper(*args, **kwargs): return wrapper +@retry +def remove_dimensions_from_bands(image_ee): + # first delete dimensions key from dictionary + # otherwise the entire image is extracted (don't know why) + try: + im_bands = image_ee.getInfo()["bands"] + for j in range(len(im_bands)): + del im_bands[j]["dimensions"] + return im_bands + except Exception as e: + print(f"An error occurred: {e}") + raise + + def get_image_quality(satname: str, im_meta: dict) -> str: """ Get the image quality for a given satellite name and image metadata. @@ -450,9 +464,7 @@ def retrieve_images( # first delete dimensions key from dictionary # otherwise the entire image is extracted (don't know why) - im_bands = image_ee.getInfo()["bands"] - for j in range(len(im_bands)): - del im_bands[j]["dimensions"] + im_bands = remove_dimensions_from_bands(image_ee) # =============================================================================================# # Landsat 5 download @@ -1166,6 +1178,7 @@ def get_s2cloudless(image_list: list, inputs: dict): return matched_cloud_images +@retry def get_image_info(collection, satname, polygon, dates, **kwargs): """ Reads info about EE images for the specified collection, satellite and dates @@ -1513,7 +1526,7 @@ def filter_S2_collection(im_list): return im_list else: idx_max = np.argmax([np.sum(utm_zones == _) for _ in np.unique(utm_zones)]) - utm_zone_selected = np.unique(utm_zones)[idx_max] + utm_zone_selected = np.unique(utm_zones)[idx_max] # find the images that were acquired at the same time but have different utm zones idx_all = np.arange(0, len(im_list), 1) idx_covered = np.ones(len(im_list)).astype(bool) From ba92e0dab0f1b82e2eaa700ad27cb4a80b0ed550 Mon Sep 17 00:00:00 2001 From: Sharon Fitzpatrick Date: Mon, 13 Nov 2023 09:36:45 -0800 Subject: [PATCH 2/3] #7 update get_image_info no loop forever --- src/coastsat/SDS_download.py | 39 ++++++++++++++---------------------- 1 file changed, 15 insertions(+), 24 deletions(-) diff --git a/src/coastsat/SDS_download.py b/src/coastsat/SDS_download.py index f749806..329cd32 100644 --- a/src/coastsat/SDS_download.py +++ b/src/coastsat/SDS_download.py @@ -14,7 +14,6 @@ from typing import List, Dict, Union, Tuple import time from functools import wraps -import traceback # earth engine module import ee @@ -29,7 +28,6 @@ # additional modules from datetime import datetime, timedelta import pytz -import pickle from skimage import morphology, transform from scipy import ndimage @@ -121,7 +119,7 @@ def retry(func): def wrapper(*args, **kwargs): max_tries = 3 image_id = kwargs.get( - "image_id", "Unknown" + "image_id", "Unknown image id" ) # Get image_id from kwargs or use 'Unknown' for i in range(max_tries): try: @@ -1178,10 +1176,10 @@ def get_s2cloudless(image_list: list, inputs: dict): return matched_cloud_images -@retry +@retry # Apply the retry decorator to the function def get_image_info(collection, satname, polygon, dates, **kwargs): """ - Reads info about EE images for the specified collection, satellite and dates + Reads info about EE images for the specified collection, satellite, and dates KV WRL 2022 @@ -1201,25 +1199,18 @@ def get_image_info(collection, satname, polygon, dates, **kwargs): im_list: list of ee.Image objects list with the info for the images """ - while True: - try: - # get info about images - ee_col = ee.ImageCollection(collection) - # Initialize the collection with filterBounds and filterDate - col = ee_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate( - dates[0], dates[1] - ) - # If "S2tile" key is in kwargs and its associated value is truthy (not an empty string, None, etc.), - # then apply an additional filter to the collection. - if kwargs.get("S2tile"): - col = col.filterMetadata( - "MGRS_TILE", "equals", kwargs["S2tile"] - ) # 58GGP - print(f"Only keeping user-defined S2tile: {kwargs['S2tile']}") - im_list = col.getInfo().get("features") - break - except: - continue + # get info about images + ee_col = ee.ImageCollection(collection) + # Initialize the collection with filterBounds and filterDate + col = ee_col.filterBounds(ee.Geometry.Polygon(polygon)).filterDate( + dates[0], dates[1] + ) + # If "S2tile" key is in kwargs and its associated value is truthy (not an empty string, None, etc.), + # then apply an additional filter to the collection. + if kwargs.get("S2tile"): + col = col.filterMetadata("MGRS_TILE", "equals", kwargs["S2tile"]) # 58GGP + print(f"Only keeping user-defined S2tile: {kwargs['S2tile']}") + im_list = col.getInfo().get("features") # remove very cloudy images (>95% cloud cover) im_list = remove_cloudy_images(im_list, satname) return im_list From deb2785a385b9e577532b292549c9527296f524c Mon Sep 17 00:00:00 2001 From: Sharon Fitzpatrick Date: Mon, 13 Nov 2023 09:37:09 -0800 Subject: [PATCH 3/3] # 7 add test_check_images_available.py --- test_check_images_available.py | 79 ++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 test_check_images_available.py diff --git a/test_check_images_available.py b/test_check_images_available.py new file mode 100644 index 0000000..fdb8784 --- /dev/null +++ b/test_check_images_available.py @@ -0,0 +1,79 @@ +# ==========================================================# +# Shoreline extraction from satellite images +# ==========================================================# + +# Kilian Vos WRL 2018 + +# %% 1. Initial settings + +# load modules +import os +import numpy as np +import pickle +import warnings + +warnings.filterwarnings("ignore") +import matplotlib.pyplot as plt +from matplotlib import gridspec + +plt.ion() +import pandas as pd +from scipy import interpolate +from scipy import stats +from datetime import datetime, timedelta +import pytz +from coastsat import ( + SDS_download, + SDS_preprocess, + SDS_shoreline, + SDS_tools, + SDS_transects, +) + +# region of interest (longitude, latitude in WGS84) +polygon = [ + [ + [-117.51525453851755, 33.29929946257779], + [-117.51549193684221, 33.33963611101142], + [-117.46743525920786, 33.33982627488089], + [-117.46721999025168, 33.2994893366537], + [-117.51525453851755, 33.29929946257779], + ] +] +# can also be loaded from a .kml polygon +# kml_polygon = os.path.join(os.getcwd(), 'examples', 'NARRA_polygon.kml') +# polygon = SDS_tools.polygon_from_kml(kml_polygon) +# convert polygon to a smallest rectangle (sides parallel to coordinate axes) +# polygon = SDS_tools.smallest_rectangle(polygon) + +# date range +# dates = ["1984-01-01", "2022-01-01"] +# dates = ["2022-01-01", "2022-12-01"] +# dates = ["2014-06-01", "2015-03-01"] +dates = ["2014-04-29", "2014-06-01"] + + +# satellite missions +# sat_list = ["L5", "L7", "L8"] +# sat_list = ["L5", "L7", "L8", "L9", "S2"] +sat_list = [ + "L8", +] +collection = "C02" # choose Landsat collection 'C01' or 'C02' +# name of the site +sitename = "l8_failure2" +# filepath where data will be stored +filepath_data = os.path.join(os.getcwd(), "data") + +# put all the inputs into a dictionnary +inputs = { + "polygon": polygon, + "dates": dates, + "sat_list": sat_list, + "sitename": sitename, + "filepath": filepath_data, + "landsat_collection": collection, + "include_T2": True, +} + +SDS_download.check_images_available(inputs)