From fe3813207272a8b7dd392ae62ec08d796079ea48 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Thu, 8 Feb 2024 11:13:02 -0800 Subject: [PATCH 01/12] STAC DataFrame Progress --- notebooks/Catalog Tutorial.ipynb | 2 +- src/gval/accessors/gval_xarray.py | 9 +- src/gval/comparison/tabulation.py | 9 ++ src/gval/utils/loading_datasets.py | 188 ++++++++++++++++++++++++++++- tests/cases_catalogs.py | 127 +++++++++++++++++++ tests/test_catalogs.py | 50 ++++++++ 6 files changed, 375 insertions(+), 10 deletions(-) diff --git a/notebooks/Catalog Tutorial.ipynb b/notebooks/Catalog Tutorial.ipynb index 61ab3cba..9a66d022 100644 --- a/notebooks/Catalog Tutorial.ipynb +++ b/notebooks/Catalog Tutorial.ipynb @@ -884,7 +884,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/src/gval/accessors/gval_xarray.py b/src/gval/accessors/gval_xarray.py index 4d753d1a..a2631009 100644 --- a/src/gval/accessors/gval_xarray.py +++ b/src/gval/accessors/gval_xarray.py @@ -93,15 +93,18 @@ def __handle_attribute_tracking( else: del attribute_tracking_kwargs["agreement_map"] - results = candidate_map.gval.attribute_tracking_xarray( + results = _attribute_tracking_xarray( + candidate_map=candidate_map, benchmark_map=benchmark_map, agreement_map=agreement_map, **attribute_tracking_kwargs, ) else: - results = candidate_map.gval.attribute_tracking_xarray( - benchmark_map=benchmark_map, agreement_map=agreement_map + results = _attribute_tracking_xarray( + candidate_map=candidate_map, + benchmark_map=benchmark_map, + agreement_map=agreement_map, ) return results diff --git a/src/gval/comparison/tabulation.py b/src/gval/comparison/tabulation.py index ecc856c5..34df9a59 100644 --- a/src/gval/comparison/tabulation.py +++ b/src/gval/comparison/tabulation.py @@ -243,6 +243,15 @@ def _crosstab_Datasets(agreement_map: xr.DataArray) -> DataFrame[Crosstab_df]: # loop variables previous_crosstab_df = None # initializing to avoid having unset for i, b in enumerate(agreement_variable_names): + # Pass pairing dictionary to variable if necessary + if ( + agreement_map[b].attrs.get("pairing_dictionary") is None + and agreement_map.attrs.get("pairing_dictionary") is not None + ): + agreement_map[b].attrs["pairing_dictionary"] = agreement_map.attrs[ + "pairing_dictionary" + ] + crosstab_df = _crosstab_2d_DataArrays( agreement_map=agreement_map[b], band_value=b ) diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index 2c9b7934..2f9be51c 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -8,7 +8,9 @@ from typing import Union, Optional, Tuple, Iterable from numbers import Number import ast +from json import JSONDecodeError +import pandas as pd import rioxarray as rxr import xarray as xr import numpy as np @@ -323,6 +325,58 @@ def _set_crs(stack: xr.DataArray, band_metadata: list = None) -> Number: return stack.rio.write_crs(f"EPSG:{band_metadata['epsg'].values}") +def query_stac( + url: str, + collections: str, + time: str, + query: str = None, + max_items: int = None, + intersects: dict = None, + bbox: list = None, +) -> list: + """Return items from a stac query + + Parameters + ---------- + url : str + Address hosting the STAC API + collections : Union[str, list] + Name of collection to get (currently limited to one) + time : str + Single or range of values to query in the time dimension + bands: list, default = None + Bands to retrieve from service + query : str, default = None + String command to filter data + max_items : int, default = None + The maximum amount of records to retrieve + intersects : dict, default = None + Dictionary representing the type of geometry and its respective coordinates + bbox : list, default = None + Coordinates to filter the spatial range of request + + Returns + ------- + list + An iterable of STAC items + + """ + + if not isinstance(collections, Iterable): + collections = [collections] + + catalog = pystac_client.Client.open(url) + + return catalog.search( + datetime=time, + collections=[collections], + max_items=max_items, + intersects=intersects, + bbox=bbox, + query=query, + ).item_collection() + + def get_stac_data( url: str, collection: str, @@ -372,17 +426,17 @@ def get_stac_data( with warnings.catch_warnings(): warnings.simplefilter("ignore") - # Call cataloging url, search, and convert to xarray - catalog = pystac_client.Client.open(url) - stac_items = catalog.search( - datetime=time, - collections=[collection], + # Call cataloging url, search, and convert to xarray + stac_items = query_stac( + url=url, + time=time, + collections=collection, max_items=max_items, intersects=intersects, bbox=bbox, query=query, - ).get_all_items() + ) stack = stackstac.stack(stac_items, resolution=resolution) @@ -439,6 +493,128 @@ def get_stac_data( return stack +def _stac_to_df(stac_items: list, assets: list = None) -> pd.DataFrame: + """Convert a list of stac items to a DataFrame + + Parameters + ---------- + stac_items: list + List of stac items to create a catalog with + assets : list, default = None + Assets to keep, (keep all if None) + + Returns + ------- + pd.DataFrame + A DataFrame with rows for each unique item/asset combination + + Raises + ------ + ValueError + No entries in DataFrame due to nonexistent asset + + """ + + dfs, compare_idx = [], 1 + for stac_item in stac_items: + map_name, map_id, compare_id = [], [], [] + for key, item in stac_item.assets.items(): + if assets is None or key in assets: + map_name.append(key) + map_id.append(item.href) + compare_id.append(compare_idx) + compare_idx += 1 + + len_assets = len(map_name) + + df_contents = { + "collection_id": [stac_item.collection_id] * len_assets, + "item_id": [stac_item.id] * len_assets, + "item_time": [stac_item.get_datetime()] * len_assets, + "create_time": [stac_item.properties["created"]] * len_assets, + "map_id": map_id, + "map_name": map_name, + "compare_id": compare_id, + "coverage_geometry_type": [stac_item.geometry["type"]] * len_assets, + "coverage_geometry_coords": [stac_item.geometry["coordinates"]] + * len_assets, + "coverage_epsg": ["4326"] * len_assets, + "asset_epsg": [stac_item.properties["proj:epsg"]] * len_assets, + } + + dfs.append(pd.DataFrame(df_contents)) + + combined_df = pd.concat(dfs) + if combined_df.empty(): + raise ValueError("No entries in DataFrame due to nonexistent asset") + + return combined_df + + +def stac_catalog( + url: str, + collections: Union[str, list], + time: str, + query: str = None, + max_items: int = None, + intersects: dict = None, + bbox: list = None, + assets: list = None, +) -> pd.DataFrame: + """Create a STAC Catalog from a STAC query + + Parameters + ---------- + url : str + Address hosting the STAC API + collections : Union[str, list] + Name of collection/s to get + time : str + Single or range of values to query in the time dimension + query : str, default = None + String command to filter data + max_items : int, default = None + The maximum amount of records to retrieve + intersects : dict, default = None + Dictionary representing the type of geometry and its respective coordinates + bbox : list, default = None + Coordinates to filter the spatial range of request + assets : list, default = None + Assets to keep, (keep all if None) + + Returns + ------- + pd.DataFrame + DataFrame representing a catalog based on STAC query + + Raises + ------ + JSONDecodeError + Unable to make STAC query + ValueError + No items returned from query + + """ + + try: + stac_items = query_stac( + url=url, + time=time, + collections=collections, + max_items=max_items, + intersects=intersects, + bbox=bbox, + query=query, + ) + except JSONDecodeError as e: + raise e("Unable to make STAC query") + + if len(stac_items) == 0: + raise ValueError("No items returned from query") + + return _stac_to_df(stac_items, assets).reset_index(drop=True) + + def _create_circle_mask( sizes: int | Tuple[int], center: Tuple[Number, Number], radius: Number ) -> np.ndarray: diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index e216d980..13c3d935 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -7,9 +7,11 @@ from pytest_cases import parametrize import pandas as pd +from pandas import Timestamp from dask import dataframe as dd import xarray as xr import numpy as np +from json import JSONDecodeError from tests.conftest import TEST_DATA_DIR @@ -461,3 +463,128 @@ def case_compare_catalogs_fail( agreement_map_field, expected_exception, ) + + +url = "https://earth-search.aws.element84.com/v1" +collection = "sentinel-2-l2a" +times = ["2020-04-01", "2020-04-03"] +bbox = [-105.78, 35.79, -105.72, 35.84] +assets = ["aot"] + +expected_stac_df = { + 0: [ + "sentinel-2-l2a", + "S2B_13SDV_20200401_1_L2A", + Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), + "2023-10-07T12:09:07.273Z", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "aot", + 1, + "Polygon", + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.0982941692468, 35.150822790058335], + [-105.85393882465051, 35.15385918076215], + [-105.68102037327243, 35.69893282033011], + [-105.59450557216445, 35.97641506053815], + [-105.56226433775963, 36.07584727804726], + [-105.53827534157463, 36.14367977962539], + [-106.11191385205835, 36.13972769324406], + ] + ], + "4326", + 32613, + "sentinel-2-l2a", + "S2A_13SDV_20200403_1_L2A", + Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), + "2023-10-08T00:32:51.304Z", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "aot", + "Polygon", + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.09828205302668, 35.1499211736146], + [-104.89285176524281, 35.154851672138626], + [-104.89152152018616, 36.14484027029347], + [-106.11191385205835, 36.13972769324406], + ] + ], + "4326", + 32613, + "1", + -1.449866533279419, + 25.393386840820312, + 0.2044084370136261, + ], + 1: [ + "sentinel-2-l2a", + "S2B_13SDV_20200401_0_L2A", + Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), + "2022-11-06T10:14:16.681Z", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + "aot", + 2, + "Polygon", + [ + [ + [-106.11191385205835, 36.13972769324406], + [-105.53527335203748, 36.14369323431527], + [-105.56579262097148, 36.05653228802631], + [-105.68980719734964, 35.659112338538634], + [-105.85157080324588, 35.15190642354915], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], + "4326", + 32613, + "sentinel-2-l2a", + "S2A_13SDV_20200403_0_L2A", + Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), + "2022-11-06T07:21:36.990Z", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + "aot", + "Polygon", + [ + [ + [-106.11191385205835, 36.13972769324406], + [-104.89152152018616, 36.14484027029347], + [-104.89285176524281, 35.154851672138626], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], + "4326", + 32613, + "1", + -0.46196842193603516, + 11.947012901306152, + 0.15074075758457184, + ], +} + + +@parametrize( + "url, collection, times, bbox, assets, expected_catalog_df", + list(zip([url], [collection], [times], [bbox], [assets], [expected_stac_df])), +) +def stac_catalog_comparison_success( + url, collection, times, bbox, assets, expected_catalog_df +): + return (url, collection, times, bbox, assets, pd.DataFrame(expected_catalog_df)) + + +bad_url = "https://google.com" +bad_times = ["2018-04-01", "2020-04-03", "2018-04-01"] +bad_assets = ["aot", "aot", "surface_water"] +exceptions = [JSONDecodeError, ValueError, ValueError] + + +@parametrize( + "url, collection, time, bbox, assets, exception", + list(zip([bad_url, url, url], [collection] * 3, bad_times, bbox * 3, bad_assets)), +) +def stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): + return (url, collection, time, bbox, assets, exception) diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index a67ac991..d61479e0 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -12,6 +12,7 @@ from tests.conftest import _attributes_to_string from gval.catalogs.catalogs import catalog_compare +from gval.utils.loading_datasets import stac_catalog @parametrize_with_cases( @@ -219,3 +220,52 @@ def test_compare_catalogs_fail( open_kwargs=open_kwargs, agreement_map_field=agreement_map_field, ) + + +@parametrize_with_cases( + "url, collection, times, bbox, assets, expected_catalog_df", + glob="stac_catalog_comparison_success", +) +def test_stac_catalog_comparison_success( + url, collection, times, bbox, assets, expected_catalog_df +): + candidate_catalog = stac_catalog( + url=url, collections=collection, time=times[0], bbox=bbox, assets=assets + ) + benchmark_catalog = stac_catalog( + url=url, collections=collection, time=times[1], bbox=bbox, assets=assets + ) + + arguments = { + "candidate_catalog": candidate_catalog, + "benchmark_catalog": benchmark_catalog, + "on": "compare_id", + "map_ids": "map_id", + "how": "inner", + "compare_type": "continuous", + "compare_kwargs": { + "metrics": ( + "coefficient_of_determination", + "mean_absolute_error", + "mean_absolute_percentage_error", + ), + "encode_nodata": True, + "nodata": -9999, + }, + "open_kwargs": {"mask_and_scale": True, "masked": True}, + } + + stac_clog = catalog_compare(**arguments) + + assert stac_clog.equals(expected_catalog_df) + + +@parametrize_with_cases( + "url, collection, time, bbox, assets, exception", + glob="stac_catalog_comparison_fail", +) +def test_stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): + with raises(exception): + _ = stac_catalog( + url=url, collections=collection, time=time, bbox=bbox, assets=assets + ) From 3e1540ca18f065c4f8cfb488e9a7e4e0c7dba51e Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Thu, 8 Feb 2024 15:16:41 -0500 Subject: [PATCH 02/12] Finish testing for stac catalogs --- notebooks/Catalog Tutorial.ipynb | 2 +- src/gval/utils/loading_datasets.py | 24 +++--- tests/cases_catalogs.py | 120 ++++++++++++++--------------- tests/test_catalogs.py | 4 +- 4 files changed, 70 insertions(+), 80 deletions(-) diff --git a/notebooks/Catalog Tutorial.ipynb b/notebooks/Catalog Tutorial.ipynb index 9a66d022..61ab3cba 100644 --- a/notebooks/Catalog Tutorial.ipynb +++ b/notebooks/Catalog Tutorial.ipynb @@ -884,7 +884,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.10" } }, "nbformat": 4, diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index 2f9be51c..f947e88b 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -8,7 +8,6 @@ from typing import Union, Optional, Tuple, Iterable from numbers import Number import ast -from json import JSONDecodeError import pandas as pd import rioxarray as rxr @@ -545,7 +544,7 @@ def _stac_to_df(stac_items: list, assets: list = None) -> pd.DataFrame: dfs.append(pd.DataFrame(df_contents)) combined_df = pd.concat(dfs) - if combined_df.empty(): + if combined_df.empty: raise ValueError("No entries in DataFrame due to nonexistent asset") return combined_df @@ -596,18 +595,15 @@ def stac_catalog( """ - try: - stac_items = query_stac( - url=url, - time=time, - collections=collections, - max_items=max_items, - intersects=intersects, - bbox=bbox, - query=query, - ) - except JSONDecodeError as e: - raise e("Unable to make STAC query") + stac_items = query_stac( + url=url, + time=time, + collections=collections, + max_items=max_items, + intersects=intersects, + bbox=bbox, + query=query, + ) if len(stac_items) == 0: raise ValueError("No items returned from query") diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index 13c3d935..f3804d27 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -472,15 +472,21 @@ def case_compare_catalogs_fail( assets = ["aot"] expected_stac_df = { - 0: [ - "sentinel-2-l2a", - "S2B_13SDV_20200401_1_L2A", + "collection_id_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], + "item_id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], + "item_time_candidate": [ Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), - "2023-10-07T12:09:07.273Z", + Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), + ], + "create_time_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "map_id_candidate": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "aot", - 1, - "Polygon", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "map_name_candidate": ["aot", "aot"], + "compare_id": [1, 2], + "coverage_geometry_type_candidate": ["Polygon", "Polygon"], + "coverage_geometry_coords_candidate": [ [ [ [-106.11191385205835, 36.13972769324406], @@ -493,60 +499,43 @@ def case_compare_catalogs_fail( [-106.11191385205835, 36.13972769324406], ] ], - "4326", - 32613, - "sentinel-2-l2a", - "S2A_13SDV_20200403_1_L2A", - Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), - "2023-10-08T00:32:51.304Z", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "aot", - "Polygon", [ [ [-106.11191385205835, 36.13972769324406], + [-105.53527335203748, 36.14369323431527], + [-105.56579262097148, 36.05653228802631], + [-105.68980719734964, 35.659112338538634], + [-105.85157080324588, 35.15190642354915], [-106.09828205302668, 35.1499211736146], - [-104.89285176524281, 35.154851672138626], - [-104.89152152018616, 36.14484027029347], [-106.11191385205835, 36.13972769324406], ] ], - "4326", - 32613, - "1", - -1.449866533279419, - 25.393386840820312, - 0.2044084370136261, ], - 1: [ - "sentinel-2-l2a", - "S2B_13SDV_20200401_0_L2A", - Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), - "2022-11-06T10:14:16.681Z", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - "aot", - 2, - "Polygon", + "coverage_epsg_candidate": ["4326", "4326"], + "asset_epsg_candidate": [32613, 32613], + "collection_id_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], + "item_id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], + "item_time_benchmark": [ + Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), + Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), + ], + "create_time_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "map_name_benchmark": ["aot", "aot"], + "coverage_geometry_type_benchmark": ["Polygon", "Polygon"], + "coverage_geometry_coords_benchmark": [ [ [ [-106.11191385205835, 36.13972769324406], - [-105.53527335203748, 36.14369323431527], - [-105.56579262097148, 36.05653228802631], - [-105.68980719734964, 35.659112338538634], - [-105.85157080324588, 35.15190642354915], [-106.09828205302668, 35.1499211736146], + [-104.89285176524281, 35.154851672138626], + [-104.89152152018616, 36.14484027029347], [-106.11191385205835, 36.13972769324406], ] ], - "4326", - 32613, - "sentinel-2-l2a", - "S2A_13SDV_20200403_0_L2A", - Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), - "2022-11-06T07:21:36.990Z", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - "aot", - "Polygon", [ [ [-106.11191385205835, 36.13972769324406], @@ -556,35 +545,38 @@ def case_compare_catalogs_fail( [-106.11191385205835, 36.13972769324406], ] ], - "4326", - 32613, - "1", - -0.46196842193603516, - 11.947012901306152, - 0.15074075758457184, ], + "coverage_epsg_benchmark": ["4326", "4326"], + "asset_epsg_benchmark": [32613, 32613], + "band": ["1", "1"], + "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], + "mean_absolute_error": [25.393386840820312, 11.947012901306152], + "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], } -@parametrize( - "url, collection, times, bbox, assets, expected_catalog_df", - list(zip([url], [collection], [times], [bbox], [assets], [expected_stac_df])), -) -def stac_catalog_comparison_success( - url, collection, times, bbox, assets, expected_catalog_df -): - return (url, collection, times, bbox, assets, pd.DataFrame(expected_catalog_df)) +def case_stac_catalog_comparison_success(): + return url, collection, times, bbox, assets, pd.DataFrame(expected_stac_df) bad_url = "https://google.com" -bad_times = ["2018-04-01", "2020-04-03", "2018-04-01"] +bad_times = ["2018-04-01", "1940-04-03", "2020-04-01"] bad_assets = ["aot", "aot", "surface_water"] exceptions = [JSONDecodeError, ValueError, ValueError] @parametrize( "url, collection, time, bbox, assets, exception", - list(zip([bad_url, url, url], [collection] * 3, bad_times, bbox * 3, bad_assets)), + list( + zip( + [bad_url, url, url], + [collection] * 3, + bad_times, + [bbox] * 3, + bad_assets, + exceptions, + ) + ), ) -def stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): - return (url, collection, time, bbox, assets, exception) +def case_stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): + return url, collection, time, bbox, assets, exception diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index d61479e0..bc4c75b2 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -257,7 +257,9 @@ def test_stac_catalog_comparison_success( stac_clog = catalog_compare(**arguments) - assert stac_clog.equals(expected_catalog_df) + pd.testing.assert_frame_equal( + stac_clog, expected_catalog_df, check_dtype=False, check_index_type=False + ), "Computed catalog did not match the expected catalog df" @parametrize_with_cases( From b48ec7d8d650fe260a53f065a794b1eea39c3812 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Thu, 8 Feb 2024 15:24:03 -0500 Subject: [PATCH 03/12] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index eb308926..40a909b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ authors = [ requires-python = ">=3.8" keywords = ["geospatial", "evaluations"] license = {text = "MIT"} -version = "0.2.5" +version = "0.2.6" dynamic = ["readme", "dependencies"] [project.optional-dependencies] From a790a75c770f75c88818ff36abeb091aa1824ced Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Thu, 29 Feb 2024 12:26:52 -0500 Subject: [PATCH 04/12] Remove wrapper for pystac_client queries --- src/gval/utils/loading_datasets.py | 327 +++++++++-------------------- tests/cases_catalogs.py | 276 +++++++++++++++++------- tests/cases_stac.py | 22 +- tests/test_catalogs.py | 38 +++- tests/test_stac.py | 38 ++-- 5 files changed, 362 insertions(+), 339 deletions(-) diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index f947e88b..cf9a3fc3 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -13,10 +13,11 @@ import rioxarray as rxr import xarray as xr import numpy as np +from shapely.geometry import MultiPoint, shape from tempfile import NamedTemporaryFile from rio_cogeo.cogeo import cog_translate from rio_cogeo.profiles import cog_profiles -import pystac_client +from pystac.item_collection import ItemCollection import stackstac @@ -324,93 +325,23 @@ def _set_crs(stack: xr.DataArray, band_metadata: list = None) -> Number: return stack.rio.write_crs(f"EPSG:{band_metadata['epsg'].values}") -def query_stac( - url: str, - collections: str, - time: str, - query: str = None, - max_items: int = None, - intersects: dict = None, - bbox: list = None, -) -> list: - """Return items from a stac query - - Parameters - ---------- - url : str - Address hosting the STAC API - collections : Union[str, list] - Name of collection to get (currently limited to one) - time : str - Single or range of values to query in the time dimension - bands: list, default = None - Bands to retrieve from service - query : str, default = None - String command to filter data - max_items : int, default = None - The maximum amount of records to retrieve - intersects : dict, default = None - Dictionary representing the type of geometry and its respective coordinates - bbox : list, default = None - Coordinates to filter the spatial range of request - - Returns - ------- - list - An iterable of STAC items - - """ - - if not isinstance(collections, Iterable): - collections = [collections] - - catalog = pystac_client.Client.open(url) - - return catalog.search( - datetime=time, - collections=[collections], - max_items=max_items, - intersects=intersects, - bbox=bbox, - query=query, - ).item_collection() - - def get_stac_data( - url: str, - collection: str, - time: str, + stac_items: ItemCollection, bands: list = None, - query: str = None, time_aggregate: str = None, - max_items: int = None, - intersects: dict = None, - bbox: list = None, resolution: int = None, nodata_fill: Number = None, ) -> xr.Dataset: - """ + """Transform STAC Items in to an xarray object Parameters ---------- - url : str - Address hosting the STAC API - collection : str - Name of collection to get (currently limited to one) - time : str - Single or range of values to query in the time dimension + stac_items : ItemCollection + STAC Item Collection returned from pystac client bands: list, default = None Bands to retrieve from service - query : str, default = None - String command to filter data time_aggregate : str, default = None Method to aggregate multiple time stamps - max_items : int, default = None - The maximum amount of records to retrieve - intersects : dict, default = None - Dictionary representing the type of geometry and its respective coordinates - bbox : list, default = None - Coordinates to filter the spatial range of request resolution : int, default = 10 Resolution to get data from nodata_fill : Number, default = None @@ -421,84 +352,77 @@ def get_stac_data( xr.Dataset Xarray object with resepective STAC API data + Raises + ------ + ValueError + A valid aggregate must be used for time ranges + """ with warnings.catch_warnings(): warnings.simplefilter("ignore") - - # Call cataloging url, search, and convert to xarray - stac_items = query_stac( - url=url, - time=time, - collections=collection, - max_items=max_items, - intersects=intersects, - bbox=bbox, - query=query, - ) - stack = stackstac.stack(stac_items, resolution=resolution) - # Only get unique time indices in case there are duplicates - _, idxs = np.unique(stack.coords["time"], return_index=True) - stack = stack[idxs] - - # Aggregate if there is more than one time - if stack.coords["time"].shape[0] > 1: - crs = stack.rio.crs - if time_aggregate == "mean": - stack = stack.mean(dim="time") - stack.attrs["time_aggregate"] = "mean" - elif time_aggregate == "min": - stack = stack.min(dim="time") - stack.attrs["time_aggregate"] = "min" - elif time_aggregate == "max": - stack = stack.max(dim="time") - stack.attrs["time_aggregate"] = "max" - else: - raise ValueError("A valid aggregate must be used for time ranges") - - stack.rio.write_crs(crs, inplace=True) + # Only get unique time indices in case there are duplicates + _, idxs = np.unique(stack.coords["time"], return_index=True) + stack = stack[idxs] + + # Aggregate if there is more than one time + if stack.coords["time"].shape[0] > 1: + crs = stack.rio.crs + if time_aggregate == "mean": + stack = stack.mean(dim="time") + stack.attrs["time_aggregate"] = "mean" + elif time_aggregate == "min": + stack = stack.min(dim="time") + stack.attrs["time_aggregate"] = "min" + elif time_aggregate == "max": + stack = stack.max(dim="time") + stack.attrs["time_aggregate"] = "max" else: - stack = stack[0] - stack.attrs["time_aggregate"] = "none" + raise ValueError("A valid aggregate must be used for time ranges") - # Select specific bands - if bands is not None: - bands = [bands] if isinstance(bands, str) else bands - stack = stack.sel({"band": bands}) + stack.rio.write_crs(crs, inplace=True) + else: + stack = stack[0] + stack.attrs["time_aggregate"] = "none" - band_metadata = ( - stack.coords["raster:bands"] if "raster:bands" in stack.coords else None - ) - if "band" in stack.dims: - og_names = [name for name in stack.coords["band"]] - names = [f"band_{x + 1}" for x in range(len(stack.coords["band"]))] - stack = stack.assign_coords({"band": names}).to_dataset(dim="band") + # Select specific bands + if bands is not None: + bands = [bands] if isinstance(bands, str) else bands + stack = stack.sel({"band": bands}) - for metadata, var, og_var in zip(band_metadata, stack.data_vars, og_names): - _set_nodata(stack[var], metadata, nodata_fill) - stack[var] = _set_crs(stack[var], band_metadata) - stack[var].attrs["original_name"] = og_var + band_metadata = ( + stack.coords["raster:bands"] if "raster:bands" in stack.coords else None + ) + if "band" in stack.dims: + og_names = [name for name in stack.coords["band"]] + names = [f"band_{x + 1}" for x in range(len(stack.coords["band"]))] + stack = stack.assign_coords({"band": names}).to_dataset(dim="band") - else: - stack = stack.to_dataset(name="band_1") - _set_nodata(stack["band_1"], band_metadata, nodata_fill) - stack["band_1"] = _set_crs(stack["band_1"]) - stack["band_1"].attrs["original_name"] = ( - bands[0] if isinstance(bands, list) else bands - ) + for metadata, var, og_var in zip(band_metadata, stack.data_vars, og_names): + _set_nodata(stack[var], metadata, nodata_fill) + stack[var] = _set_crs(stack[var], band_metadata) + stack[var].attrs["original_name"] = og_var + + else: + stack = stack.to_dataset(name="band_1") + _set_nodata(stack["band_1"], band_metadata, nodata_fill) + stack["band_1"] = _set_crs(stack["band_1"]) + stack["band_1"].attrs["original_name"] = ( + bands[0] if isinstance(bands, list) else bands + ) - return stack + return stack -def _stac_to_df(stac_items: list, assets: list = None) -> pd.DataFrame: - """Convert a list of stac items to a DataFrame +def stac_to_df(stac_items: ItemCollection, assets: list = None) -> pd.DataFrame: + """Convert STAC Items in to a DataFrame Parameters ---------- - stac_items: list - List of stac items to create a catalog with + stac_items: ItemCollection + STAC Item Collection returned from pystac client assets : list, default = None Assets to keep, (keep all if None) @@ -514,101 +438,50 @@ def _stac_to_df(stac_items: list, assets: list = None) -> pd.DataFrame: """ - dfs, compare_idx = [], 1 - for stac_item in stac_items: - map_name, map_id, compare_id = [], [], [] - for key, item in stac_item.assets.items(): - if assets is None or key in assets: - map_name.append(key) - map_id.append(item.href) - compare_id.append(compare_idx) + item_dfs, compare_idx = [], 1 + + # Iterate through each STAC Item + for item in stac_items: + item_dict = item.to_dict() + item_columns = {} + + # Get columns for all collection level and item level properties + for key, val in item_dict["properties"].items(): + if not isinstance(val, list): + if isinstance(val, dict): + for k, v in val.items(): + item_columns[k] = [v] + else: + item_columns[key] = [val] + + item_columns["bbox"] = MultiPoint(np.array(item_dict["bbox"]).reshape(2, 2)).wkt + item_columns["geometry"] = shape(item_dict["geometry"]).wkt + + unique_keys = [] + for k, v in item_dict["assets"].items(): + for key in v.keys(): + if key not in unique_keys: + unique_keys.append(key) + + # Create new row for each asset with and assign compare_id and map_id + asset_dfs = [] + for k, v in item_dict["assets"].items(): + if assets is None or k in assets: + asset_columns = item_columns.copy() + + asset_columns[key] = [str(v.get(key, "N/a"))] + asset_columns["compare_id"] = compare_idx + asset_columns["map_id"] = v["href"] compare_idx += 1 + asset_columns["asset"] = [k] + for key in unique_keys: + asset_columns[key] = [str(v.get(key, "N/a"))] - len_assets = len(map_name) - - df_contents = { - "collection_id": [stac_item.collection_id] * len_assets, - "item_id": [stac_item.id] * len_assets, - "item_time": [stac_item.get_datetime()] * len_assets, - "create_time": [stac_item.properties["created"]] * len_assets, - "map_id": map_id, - "map_name": map_name, - "compare_id": compare_id, - "coverage_geometry_type": [stac_item.geometry["type"]] * len_assets, - "coverage_geometry_coords": [stac_item.geometry["coordinates"]] - * len_assets, - "coverage_epsg": ["4326"] * len_assets, - "asset_epsg": [stac_item.properties["proj:epsg"]] * len_assets, - } - - dfs.append(pd.DataFrame(df_contents)) - - combined_df = pd.concat(dfs) - if combined_df.empty: - raise ValueError("No entries in DataFrame due to nonexistent asset") - - return combined_df - - -def stac_catalog( - url: str, - collections: Union[str, list], - time: str, - query: str = None, - max_items: int = None, - intersects: dict = None, - bbox: list = None, - assets: list = None, -) -> pd.DataFrame: - """Create a STAC Catalog from a STAC query - - Parameters - ---------- - url : str - Address hosting the STAC API - collections : Union[str, list] - Name of collection/s to get - time : str - Single or range of values to query in the time dimension - query : str, default = None - String command to filter data - max_items : int, default = None - The maximum amount of records to retrieve - intersects : dict, default = None - Dictionary representing the type of geometry and its respective coordinates - bbox : list, default = None - Coordinates to filter the spatial range of request - assets : list, default = None - Assets to keep, (keep all if None) - - Returns - ------- - pd.DataFrame - DataFrame representing a catalog based on STAC query - - Raises - ------ - JSONDecodeError - Unable to make STAC query - ValueError - No items returned from query - - """ - - stac_items = query_stac( - url=url, - time=time, - collections=collections, - max_items=max_items, - intersects=intersects, - bbox=bbox, - query=query, - ) + asset_dfs.append(pd.DataFrame(asset_columns)) - if len(stac_items) == 0: - raise ValueError("No items returned from query") + item_dfs.append(pd.concat(asset_dfs)) - return _stac_to_df(stac_items, assets).reset_index(drop=True) + return pd.concat(item_dfs, ignore_index=True) def _create_circle_mask( diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index f3804d27..1a857985 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -7,11 +7,9 @@ from pytest_cases import parametrize import pandas as pd -from pandas import Timestamp from dask import dataframe as dd import xarray as xr import numpy as np -from json import JSONDecodeError from tests.conftest import TEST_DATA_DIR @@ -472,82 +470,209 @@ def case_compare_catalogs_fail( assets = ["aot"] expected_stac_df = { - "collection_id_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], - "item_id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], - "item_time_candidate": [ - Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), - Timestamp("2020-04-01 18:04:04.327000+0000", tz="utc"), + "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "platform_candidate": ["sentinel-2b", "sentinel-2b"], + "constellation_candidate": ["sentinel-2", "sentinel-2"], + "eo:cloud_cover_candidate": [26.25798, 26.031335], + "proj:epsg_candidate": [32613, 32613], + "mgrs:utm_zone_candidate": [13, 13], + "mgrs:latitude_band_candidate": ["S", "S"], + "mgrs:grid_square_candidate": ["DV", "DV"], + "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], + "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], + "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], + "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], + "s2:saturated_defective_pixel_percentage_candidate": [0, 0], + "s2:dark_features_percentage_candidate": [2.557362, 4.251513], + "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], + "s2:vegetation_percentage_candidate": [6.046846, 6.331863], + "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], + "s2:water_percentage_candidate": [0.019934, 0.045153], + "s2:unclassified_percentage_candidate": [5.82255, 5.403911], + "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], + "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], + "s2:thin_cirrus_percentage_candidate": [0, 0], + "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], + "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_candidate": ["05.00", "02.14"], + "s2:product_uri_candidate": [ + "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", + "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", ], - "create_time_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "s2:generation_time_candidate": [ + "2023-06-24T05:35:45.000000Z", + "2020-04-01T22:01:55.000000Z", + ], + "s2:datatake_id_candidate": [ + "GS2B_20200401T174909_016040_N05.00", + "GS2B_20200401T174909_016040_N02.14", + ], + "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_candidate": [ + "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", + "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", + ], + "s2:granule_id_candidate": [ + "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", + "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_candidate": [1.00356283682453, 1.00356283682453], + "datetime_candidate": [ + "2020-04-01T18:04:04.327000Z", + "2020-04-01T18:04:04.327000Z", + ], + "s2:sequence_candidate": ["1", "0"], + "earthsearch:s3_path_candidate": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", + ], + "earthsearch:payload_id_candidate": [ + "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", + "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", + ], + "earthsearch:boa_offset_applied_candidate": [True, False], + "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], + "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "bbox_candidate": [ + "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", + ], + "geometry_candidate": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], + "compare_id": [1, 2], "map_id_candidate": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", ], - "map_name_candidate": ["aot", "aot"], - "compare_id": [1, 2], - "coverage_geometry_type_candidate": ["Polygon", "Polygon"], - "coverage_geometry_coords_candidate": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.0982941692468, 35.150822790058335], - [-105.85393882465051, 35.15385918076215], - [-105.68102037327243, 35.69893282033011], - [-105.59450557216445, 35.97641506053815], - [-105.56226433775963, 36.07584727804726], - [-105.53827534157463, 36.14367977962539], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-105.53527335203748, 36.14369323431527], - [-105.56579262097148, 36.05653228802631], - [-105.68980719734964, 35.659112338538634], - [-105.85157080324588, 35.15190642354915], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "coverage_epsg_candidate": ["4326", "4326"], - "asset_epsg_candidate": [32613, 32613], - "collection_id_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], - "item_id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], - "item_time_benchmark": [ - Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), - Timestamp("2020-04-03 17:54:07.524000+0000", tz="utc"), - ], - "create_time_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "asset_candidate": ["aot", "aot"], + "href_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "type_candidate": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_candidate": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_candidate": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_candidate": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_candidate": ["N/a", "N/a"], + "gsd_candidate": ["N/a", "N/a"], + "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "platform_benchmark": ["sentinel-2a", "sentinel-2a"], + "constellation_benchmark": ["sentinel-2", "sentinel-2"], + "eo:cloud_cover_benchmark": [0.394644, 0.946059], + "proj:epsg_benchmark": [32613, 32613], + "mgrs:utm_zone_benchmark": [13, 13], + "mgrs:latitude_band_benchmark": ["S", "S"], + "mgrs:grid_square_benchmark": ["DV", "DV"], + "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], + "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], + "s2:degraded_msi_data_percentage_benchmark": [0, 0], + "s2:nodata_pixel_percentage_benchmark": [0, 0], + "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], + "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], + "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], + "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], + "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], + "s2:water_percentage_benchmark": [0.095783, 0.093679], + "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], + "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], + "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], + "s2:thin_cirrus_percentage_benchmark": [0, 0], + "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], + "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_benchmark": ["05.00", "02.14"], + "s2:product_uri_benchmark": [ + "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", + "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", + ], + "s2:generation_time_benchmark": [ + "2023-05-10T05:34:45.000000Z", + "2020-04-03T22:01:05.000000Z", + ], + "s2:datatake_id_benchmark": [ + "GS2A_20200403T173901_024977_N05.00", + "GS2A_20200403T173901_024977_N02.14", + ], + "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_benchmark": [ + "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", + "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", + ], + "s2:granule_id_benchmark": [ + "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", + "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_benchmark": [1.00241535908783, 1.00241535908783], + "datetime_benchmark": [ + "2020-04-03T17:54:07.524000Z", + "2020-04-03T17:54:07.524000Z", + ], + "s2:sequence_benchmark": ["1", "0"], + "earthsearch:s3_path_benchmark": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", + ], + "earthsearch:payload_id_benchmark": [ + "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", + "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", + ], + "earthsearch:boa_offset_applied_benchmark": [True, False], + "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], + "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "bbox_benchmark": [ + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + ], + "geometry_benchmark": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], "map_id_benchmark": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", ], - "map_name_benchmark": ["aot", "aot"], - "coverage_geometry_type_benchmark": ["Polygon", "Polygon"], - "coverage_geometry_coords_benchmark": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.09828205302668, 35.1499211736146], - [-104.89285176524281, 35.154851672138626], - [-104.89152152018616, 36.14484027029347], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-104.89152152018616, 36.14484027029347], - [-104.89285176524281, 35.154851672138626], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "coverage_epsg_benchmark": ["4326", "4326"], - "asset_epsg_benchmark": [32613, 32613], + "asset_benchmark": ["aot", "aot"], + "href_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "type_benchmark": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_benchmark": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_benchmark": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_benchmark": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_benchmark": ["N/a", "N/a"], + "gsd_benchmark": ["N/a", "N/a"], "band": ["1", "1"], "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], "mean_absolute_error": [25.393386840820312, 11.947012901306152], @@ -559,20 +684,19 @@ def case_stac_catalog_comparison_success(): return url, collection, times, bbox, assets, pd.DataFrame(expected_stac_df) -bad_url = "https://google.com" -bad_times = ["2018-04-01", "1940-04-03", "2020-04-01"] -bad_assets = ["aot", "aot", "surface_water"] -exceptions = [JSONDecodeError, ValueError, ValueError] +bad_times = ["2020-04-01"] +bad_assets = ["surface_water"] +exceptions = [ValueError] @parametrize( "url, collection, time, bbox, assets, exception", list( zip( - [bad_url, url, url], - [collection] * 3, + [url], + [collection], bad_times, - [bbox] * 3, + [bbox], bad_assets, exceptions, ) diff --git a/tests/cases_stac.py b/tests/cases_stac.py index 272425fc..7335f91d 100644 --- a/tests/cases_stac.py +++ b/tests/cases_stac.py @@ -153,35 +153,29 @@ def case_stac_api_call( ) -bands_fail = [["aot"], ["aot"], ["red"]] +bands_fail = [["aot"]] time_fail = [ - ["1945-04-01", "1945-04-03"], ["2020-04-01/2020-04-03", "2020-04-06/2020-04-08"], - ["2020-04-01", "2020-04-03"], ] -time_aggreagte_fail = [None, None, None] +time_aggregate_fail = [None] -nodata_fill_fail = [ - None, - None, - None, -] +nodata_fill_fail = [None] -exceptions = [ValueError] * 3 +exceptions = [ValueError] @parametrize( "url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception", list( zip( - [url] * len(time), - [collection] * len(time), - [bbox] * len(time), + [url] * len(time_fail), + [collection] * len(time_fail), + [bbox] * len(time_fail), time_fail, bands_fail, - time_aggreagte_fail, + time_aggregate_fail, nodata_fill_fail, exceptions, ) diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index bc4c75b2..622b5456 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -9,10 +9,11 @@ import dask.dataframe as dd import rioxarray as rxr import xarray as xr +import pystac_client from tests.conftest import _attributes_to_string from gval.catalogs.catalogs import catalog_compare -from gval.utils.loading_datasets import stac_catalog +from gval.utils.loading_datasets import stac_to_df @parametrize_with_cases( @@ -229,12 +230,23 @@ def test_compare_catalogs_fail( def test_stac_catalog_comparison_success( url, collection, times, bbox, assets, expected_catalog_df ): - candidate_catalog = stac_catalog( - url=url, collections=collection, time=times[0], bbox=bbox, assets=assets - ) - benchmark_catalog = stac_catalog( - url=url, collections=collection, time=times[1], bbox=bbox, assets=assets - ) + catalog = pystac_client.Client.open(url) + + candidate_items = catalog.search( + datetime=times[0], + collections=[collection], + bbox=bbox, + ).item_collection() + + candidate_catalog = stac_to_df(stac_items=candidate_items, assets=assets) + + benchmark_items = catalog.search( + datetime=times[1], + collections=[collection], + bbox=bbox, + ).item_collection() + + benchmark_catalog = stac_to_df(stac_items=benchmark_items, assets=assets) arguments = { "candidate_catalog": candidate_catalog, @@ -268,6 +280,12 @@ def test_stac_catalog_comparison_success( ) def test_stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): with raises(exception): - _ = stac_catalog( - url=url, collections=collection, time=time, bbox=bbox, assets=assets - ) + catalog = pystac_client.Client.open(url) + + candidate_items = catalog.search( + datetime=time, + collections=[collection], + bbox=bbox, + ).item_collection() + + _ = stac_to_df(stac_items=candidate_items, assets=assets) diff --git a/tests/test_stac.py b/tests/test_stac.py index 3af1d0ba..aa0d0906 100644 --- a/tests/test_stac.py +++ b/tests/test_stac.py @@ -2,6 +2,7 @@ from pytest import raises import xarray as xr import pandas as pd +import pystac_client from gval.utils.loading_datasets import get_stac_data @@ -17,22 +18,30 @@ def test_stac_api_call( Tests call for stac API, (IDK if this data can be mocked, API calls in unit tests are dubious) """ + catalog = pystac_client.Client.open(url) + + candidate_items = catalog.search( + datetime=time[0], + collections=[collection], + bbox=bbox, + ).item_collection() + candidate = get_stac_data( - url=url, - collection=collection, - time=time[0], + stac_items=candidate_items, bands=bands, - bbox=bbox, time_aggregate=time_aggregate, nodata_fill=nodata_fill, ) + benchmark_items = catalog.search( + datetime=time[1], + collections=[collection], + bbox=bbox, + ).item_collection() + benchmark = get_stac_data( - url=url, - collection=collection, - time=time[1], + stac_items=benchmark_items, bands=bands, - bbox=bbox, time_aggregate=time_aggregate, nodata_fill=nodata_fill, ) @@ -80,12 +89,17 @@ def test_stac_api_call_fail( """ with raises(exception): + catalog = pystac_client.Client.open(url) + + candidate_items = catalog.search( + datetime=time[0], + collections=[collection], + bbox=bbox, + ).item_collection() + _ = get_stac_data( - url=url, - collection=collection, - time=time[0], + stac_items=candidate_items, bands=bands, - bbox=bbox, time_aggregate=time_aggregate, nodata_fill=nodata_fill, ) From 47682bd9c42cd106d16d22ec92d3c6fba19fcc5c Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Thu, 29 Feb 2024 16:29:11 -0500 Subject: [PATCH 05/12] Include allow and block lists for STAC DataFrames --- src/gval/utils/loading_datasets.py | 22 +- tests/cases_catalogs.py | 715 ++++++++++++++++++++--------- tests/test_catalogs.py | 31 +- 3 files changed, 537 insertions(+), 231 deletions(-) diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index cf9a3fc3..1aa4c4cd 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -416,7 +416,12 @@ def get_stac_data( return stack -def stac_to_df(stac_items: ItemCollection, assets: list = None) -> pd.DataFrame: +def stac_to_df( + stac_items: ItemCollection, + assets: list = None, + column_allow_list: list = None, + column_block_list: list = None, +) -> pd.DataFrame: """Convert STAC Items in to a DataFrame Parameters @@ -425,6 +430,10 @@ def stac_to_df(stac_items: ItemCollection, assets: list = None) -> pd.DataFrame: STAC Item Collection returned from pystac client assets : list, default = None Assets to keep, (keep all if None) + column_allow_list: list, default = None + List of columns to allow in the result DataFrame + column_block_list: list, default = None + List of columns to remove in the result DataFrame Returns ------- @@ -481,7 +490,16 @@ def stac_to_df(stac_items: ItemCollection, assets: list = None) -> pd.DataFrame: item_dfs.append(pd.concat(asset_dfs)) - return pd.concat(item_dfs, ignore_index=True) + # Concatenate the DataFrames and remove unwanted columns if allow and block lists exist + catalog_df = pd.concat(item_dfs, ignore_index=True) + + if column_allow_list is not None: + catalog_df = catalog_df[column_allow_list] + + if column_block_list is not None: + catalog_df = catalog_df.drop(column_block_list, axis=1) + + return catalog_df def _create_circle_mask( diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index 1a857985..3bc835a2 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -469,238 +469,509 @@ def case_compare_catalogs_fail( bbox = [-105.78, 35.79, -105.72, 35.84] assets = ["aot"] -expected_stac_df = { - "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "platform_candidate": ["sentinel-2b", "sentinel-2b"], - "constellation_candidate": ["sentinel-2", "sentinel-2"], - "eo:cloud_cover_candidate": [26.25798, 26.031335], - "proj:epsg_candidate": [32613, 32613], - "mgrs:utm_zone_candidate": [13, 13], - "mgrs:latitude_band_candidate": ["S", "S"], - "mgrs:grid_square_candidate": ["DV", "DV"], - "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], - "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], - "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], - "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], - "s2:saturated_defective_pixel_percentage_candidate": [0, 0], - "s2:dark_features_percentage_candidate": [2.557362, 4.251513], - "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], - "s2:vegetation_percentage_candidate": [6.046846, 6.331863], - "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], - "s2:water_percentage_candidate": [0.019934, 0.045153], - "s2:unclassified_percentage_candidate": [5.82255, 5.403911], - "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], - "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], - "s2:thin_cirrus_percentage_candidate": [0, 0], - "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], - "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_candidate": ["05.00", "02.14"], - "s2:product_uri_candidate": [ - "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", - "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", - ], - "s2:generation_time_candidate": [ - "2023-06-24T05:35:45.000000Z", - "2020-04-01T22:01:55.000000Z", - ], - "s2:datatake_id_candidate": [ - "GS2B_20200401T174909_016040_N05.00", - "GS2B_20200401T174909_016040_N02.14", - ], - "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_candidate": [ - "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", - "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", - ], - "s2:granule_id_candidate": [ - "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", - "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", - ], - "s2:reflectance_conversion_factor_candidate": [1.00356283682453, 1.00356283682453], - "datetime_candidate": [ - "2020-04-01T18:04:04.327000Z", - "2020-04-01T18:04:04.327000Z", - ], - "s2:sequence_candidate": ["1", "0"], - "earthsearch:s3_path_candidate": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", - ], - "earthsearch:payload_id_candidate": [ - "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", - "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", - ], - "earthsearch:boa_offset_applied_candidate": [True, False], - "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], - "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "bbox_candidate": [ - "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", - ], - "geometry_candidate": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], - "compare_id": [1, 2], - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "asset_candidate": ["aot", "aot"], - "href_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "type_candidate": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], - "title_candidate": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], - "proj:transform_candidate": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", - ], - "raster:bands_candidate": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - ], - "eo:bands_candidate": ["N/a", "N/a"], - "gsd_candidate": ["N/a", "N/a"], - "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "platform_benchmark": ["sentinel-2a", "sentinel-2a"], - "constellation_benchmark": ["sentinel-2", "sentinel-2"], - "eo:cloud_cover_benchmark": [0.394644, 0.946059], - "proj:epsg_benchmark": [32613, 32613], - "mgrs:utm_zone_benchmark": [13, 13], - "mgrs:latitude_band_benchmark": ["S", "S"], - "mgrs:grid_square_benchmark": ["DV", "DV"], - "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], - "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], - "s2:degraded_msi_data_percentage_benchmark": [0, 0], - "s2:nodata_pixel_percentage_benchmark": [0, 0], - "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], - "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], - "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], - "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], - "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], - "s2:water_percentage_benchmark": [0.095783, 0.093679], - "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], - "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], - "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], - "s2:thin_cirrus_percentage_benchmark": [0, 0], - "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], - "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_benchmark": ["05.00", "02.14"], - "s2:product_uri_benchmark": [ - "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", - "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", - ], - "s2:generation_time_benchmark": [ - "2023-05-10T05:34:45.000000Z", - "2020-04-03T22:01:05.000000Z", - ], - "s2:datatake_id_benchmark": [ - "GS2A_20200403T173901_024977_N05.00", - "GS2A_20200403T173901_024977_N02.14", - ], - "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_benchmark": [ - "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", - "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", - ], - "s2:granule_id_benchmark": [ - "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", - "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", - ], - "s2:reflectance_conversion_factor_benchmark": [1.00241535908783, 1.00241535908783], - "datetime_benchmark": [ - "2020-04-03T17:54:07.524000Z", - "2020-04-03T17:54:07.524000Z", - ], - "s2:sequence_benchmark": ["1", "0"], - "earthsearch:s3_path_benchmark": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", - ], - "earthsearch:payload_id_benchmark": [ - "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", - "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", - ], - "earthsearch:boa_offset_applied_benchmark": [True, False], - "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], - "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "bbox_benchmark": [ - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - ], - "geometry_benchmark": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "asset_benchmark": ["aot", "aot"], - "href_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "type_benchmark": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], - "title_benchmark": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], - "proj:transform_benchmark": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", - ], - "raster:bands_benchmark": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - ], - "eo:bands_benchmark": ["N/a", "N/a"], - "gsd_benchmark": ["N/a", "N/a"], - "band": ["1", "1"], - "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], - "mean_absolute_error": [25.393386840820312, 11.947012901306152], - "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], -} - - -def case_stac_catalog_comparison_success(): - return url, collection, times, bbox, assets, pd.DataFrame(expected_stac_df) +expected_stac_df = [ + { + "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "platform_candidate": ["sentinel-2b", "sentinel-2b"], + "constellation_candidate": ["sentinel-2", "sentinel-2"], + "eo:cloud_cover_candidate": [26.25798, 26.031335], + "proj:epsg_candidate": [32613, 32613], + "mgrs:utm_zone_candidate": [13, 13], + "mgrs:latitude_band_candidate": ["S", "S"], + "mgrs:grid_square_candidate": ["DV", "DV"], + "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], + "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], + "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], + "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], + "s2:saturated_defective_pixel_percentage_candidate": [0, 0], + "s2:dark_features_percentage_candidate": [2.557362, 4.251513], + "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], + "s2:vegetation_percentage_candidate": [6.046846, 6.331863], + "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], + "s2:water_percentage_candidate": [0.019934, 0.045153], + "s2:unclassified_percentage_candidate": [5.82255, 5.403911], + "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], + "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], + "s2:thin_cirrus_percentage_candidate": [0, 0], + "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], + "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_candidate": ["05.00", "02.14"], + "s2:product_uri_candidate": [ + "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", + "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", + ], + "s2:generation_time_candidate": [ + "2023-06-24T05:35:45.000000Z", + "2020-04-01T22:01:55.000000Z", + ], + "s2:datatake_id_candidate": [ + "GS2B_20200401T174909_016040_N05.00", + "GS2B_20200401T174909_016040_N02.14", + ], + "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_candidate": [ + "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", + "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", + ], + "s2:granule_id_candidate": [ + "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", + "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_candidate": [ + 1.00356283682453, + 1.00356283682453, + ], + "datetime_candidate": [ + "2020-04-01T18:04:04.327000Z", + "2020-04-01T18:04:04.327000Z", + ], + "s2:sequence_candidate": ["1", "0"], + "earthsearch:s3_path_candidate": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", + ], + "earthsearch:payload_id_candidate": [ + "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", + "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", + ], + "earthsearch:boa_offset_applied_candidate": [True, False], + "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], + "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "bbox_candidate": [ + "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", + ], + "geometry_candidate": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], + "compare_id": [1, 2], + "map_id_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "asset_candidate": ["aot", "aot"], + "href_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "type_candidate": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_candidate": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_candidate": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_candidate": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_candidate": ["N/a", "N/a"], + "gsd_candidate": ["N/a", "N/a"], + "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "platform_benchmark": ["sentinel-2a", "sentinel-2a"], + "constellation_benchmark": ["sentinel-2", "sentinel-2"], + "eo:cloud_cover_benchmark": [0.394644, 0.946059], + "proj:epsg_benchmark": [32613, 32613], + "mgrs:utm_zone_benchmark": [13, 13], + "mgrs:latitude_band_benchmark": ["S", "S"], + "mgrs:grid_square_benchmark": ["DV", "DV"], + "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], + "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], + "s2:degraded_msi_data_percentage_benchmark": [0, 0], + "s2:nodata_pixel_percentage_benchmark": [0, 0], + "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], + "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], + "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], + "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], + "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], + "s2:water_percentage_benchmark": [0.095783, 0.093679], + "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], + "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], + "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], + "s2:thin_cirrus_percentage_benchmark": [0, 0], + "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], + "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_benchmark": ["05.00", "02.14"], + "s2:product_uri_benchmark": [ + "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", + "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", + ], + "s2:generation_time_benchmark": [ + "2023-05-10T05:34:45.000000Z", + "2020-04-03T22:01:05.000000Z", + ], + "s2:datatake_id_benchmark": [ + "GS2A_20200403T173901_024977_N05.00", + "GS2A_20200403T173901_024977_N02.14", + ], + "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_benchmark": [ + "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", + "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", + ], + "s2:granule_id_benchmark": [ + "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", + "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_benchmark": [ + 1.00241535908783, + 1.00241535908783, + ], + "datetime_benchmark": [ + "2020-04-03T17:54:07.524000Z", + "2020-04-03T17:54:07.524000Z", + ], + "s2:sequence_benchmark": ["1", "0"], + "earthsearch:s3_path_benchmark": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", + ], + "earthsearch:payload_id_benchmark": [ + "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", + "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", + ], + "earthsearch:boa_offset_applied_benchmark": [True, False], + "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], + "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "bbox_benchmark": [ + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + ], + "geometry_benchmark": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "asset_benchmark": ["aot", "aot"], + "href_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "type_benchmark": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_benchmark": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_benchmark": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_benchmark": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_benchmark": ["N/a", "N/a"], + "gsd_benchmark": ["N/a", "N/a"], + "band": ["1", "1"], + "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], + "mean_absolute_error": [25.393386840820312, 11.947012901306152], + "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], + }, + { + "map_id_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "compare_id": [1, 2], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "band": ["1", "1"], + "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], + "mean_absolute_error": [25.393386840820312, 11.947012901306152], + "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], + }, + { + "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "platform_candidate": ["sentinel-2b", "sentinel-2b"], + "constellation_candidate": ["sentinel-2", "sentinel-2"], + "proj:epsg_candidate": [32613, 32613], + "mgrs:utm_zone_candidate": [13, 13], + "mgrs:latitude_band_candidate": ["S", "S"], + "mgrs:grid_square_candidate": ["DV", "DV"], + "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], + "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], + "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], + "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], + "s2:saturated_defective_pixel_percentage_candidate": [0, 0], + "s2:dark_features_percentage_candidate": [2.557362, 4.251513], + "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], + "s2:vegetation_percentage_candidate": [6.046846, 6.331863], + "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], + "s2:water_percentage_candidate": [0.019934, 0.045153], + "s2:unclassified_percentage_candidate": [5.82255, 5.403911], + "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], + "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], + "s2:thin_cirrus_percentage_candidate": [0, 0], + "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], + "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_candidate": ["05.00", "02.14"], + "s2:product_uri_candidate": [ + "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", + "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", + ], + "s2:generation_time_candidate": [ + "2023-06-24T05:35:45.000000Z", + "2020-04-01T22:01:55.000000Z", + ], + "s2:datatake_id_candidate": [ + "GS2B_20200401T174909_016040_N05.00", + "GS2B_20200401T174909_016040_N02.14", + ], + "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_candidate": [ + "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", + "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", + ], + "s2:granule_id_candidate": [ + "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", + "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_candidate": [ + 1.00356283682453, + 1.00356283682453, + ], + "datetime_candidate": [ + "2020-04-01T18:04:04.327000Z", + "2020-04-01T18:04:04.327000Z", + ], + "s2:sequence_candidate": ["1", "0"], + "earthsearch:s3_path_candidate": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", + ], + "earthsearch:payload_id_candidate": [ + "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", + "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", + ], + "earthsearch:boa_offset_applied_candidate": [True, False], + "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], + "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], + "bbox_candidate": [ + "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", + ], + "geometry_candidate": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], + "compare_id": [1, 2], + "map_id_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "asset_candidate": ["aot", "aot"], + "href_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "type_candidate": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_candidate": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_candidate": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_candidate": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_candidate": ["N/a", "N/a"], + "gsd_candidate": ["N/a", "N/a"], + "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "platform_benchmark": ["sentinel-2a", "sentinel-2a"], + "constellation_benchmark": ["sentinel-2", "sentinel-2"], + "proj:epsg_benchmark": [32613, 32613], + "mgrs:utm_zone_benchmark": [13, 13], + "mgrs:latitude_band_benchmark": ["S", "S"], + "mgrs:grid_square_benchmark": ["DV", "DV"], + "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], + "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], + "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], + "s2:degraded_msi_data_percentage_benchmark": [0, 0], + "s2:nodata_pixel_percentage_benchmark": [0, 0], + "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], + "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], + "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], + "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], + "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], + "s2:water_percentage_benchmark": [0.095783, 0.093679], + "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], + "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], + "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], + "s2:thin_cirrus_percentage_benchmark": [0, 0], + "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], + "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], + "s2:processing_baseline_benchmark": ["05.00", "02.14"], + "s2:product_uri_benchmark": [ + "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", + "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", + ], + "s2:generation_time_benchmark": [ + "2023-05-10T05:34:45.000000Z", + "2020-04-03T22:01:05.000000Z", + ], + "s2:datatake_id_benchmark": [ + "GS2A_20200403T173901_024977_N05.00", + "GS2A_20200403T173901_024977_N02.14", + ], + "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], + "s2:datastrip_id_benchmark": [ + "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", + "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", + ], + "s2:granule_id_benchmark": [ + "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", + "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", + ], + "s2:reflectance_conversion_factor_benchmark": [ + 1.00241535908783, + 1.00241535908783, + ], + "datetime_benchmark": [ + "2020-04-03T17:54:07.524000Z", + "2020-04-03T17:54:07.524000Z", + ], + "s2:sequence_benchmark": ["1", "0"], + "earthsearch:s3_path_benchmark": [ + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", + "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", + ], + "earthsearch:payload_id_benchmark": [ + "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", + "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", + ], + "earthsearch:boa_offset_applied_benchmark": [True, False], + "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], + "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], + "bbox_benchmark": [ + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", + ], + "geometry_benchmark": [ + "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", + "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", + ], + "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "asset_benchmark": ["aot", "aot"], + "href_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + ], + "type_benchmark": [ + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; application=geotiff; profile=cloud-optimized", + ], + "title_benchmark": [ + "Aerosol optical thickness (AOT)", + "Aerosol optical thickness (AOT)", + ], + "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], + "proj:transform_benchmark": [ + "[20, 0, 399960, 0, -20, 4000020]", + "[20, 0, 399960, 0, -20, 4000020]", + ], + "raster:bands_benchmark": [ + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + ], + "eo:bands_benchmark": ["N/a", "N/a"], + "gsd_benchmark": ["N/a", "N/a"], + "band": ["1", "1"], + "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], + "mean_absolute_error": [25.393386840820312, 11.947012901306152], + "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], + }, +] + +allow_list = [None, ["map_id", "compare_id"], None] +block_list = [None, None, ["eo:cloud_cover"]] + + +@parametrize( + "url, collection, times, bbox, assets, allow_list, block_list, expected_catalog_df", + list( + zip( + [url] * 3, + [collection] * 3, + [times] * 3, + [bbox] * 3, + [assets] * 3, + allow_list, + block_list, + expected_stac_df, + ) + ), +) +def case_stac_catalog_comparison_success( + url, collection, times, bbox, assets, allow_list, block_list, expected_catalog_df +): + return ( + url, + collection, + times, + bbox, + assets, + allow_list, + block_list, + pd.DataFrame(expected_catalog_df), + ) bad_times = ["2020-04-01"] -bad_assets = ["surface_water"] -exceptions = [ValueError] +bad_assets = ["surface_water", None, None] +exceptions = [ValueError, KeyError, KeyError] +bad_allow_list = [None, ["arb"], None] +bad_block_list = [None, None, ["arb"]] @parametrize( - "url, collection, time, bbox, assets, exception", + "url, collection, time, bbox, assets, allow_list, block_list, exception", list( zip( - [url], - [collection], - bad_times, - [bbox], + [url] * 3, + [collection] * 3, + bad_times * 3, + [bbox] * 3, bad_assets, + bad_allow_list, + bad_block_list, exceptions, ) ), ) -def case_stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): - return url, collection, time, bbox, assets, exception +def case_stac_catalog_comparison_fail( + url, collection, time, bbox, assets, allow_list, block_list, exception +): + return url, collection, time, bbox, assets, allow_list, block_list, exception diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index 622b5456..5392b052 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -224,11 +224,11 @@ def test_compare_catalogs_fail( @parametrize_with_cases( - "url, collection, times, bbox, assets, expected_catalog_df", + "url, collection, times, bbox, assets, allow_list, block_list, expected_catalog_df", glob="stac_catalog_comparison_success", ) def test_stac_catalog_comparison_success( - url, collection, times, bbox, assets, expected_catalog_df + url, collection, times, bbox, assets, allow_list, block_list, expected_catalog_df ): catalog = pystac_client.Client.open(url) @@ -238,7 +238,12 @@ def test_stac_catalog_comparison_success( bbox=bbox, ).item_collection() - candidate_catalog = stac_to_df(stac_items=candidate_items, assets=assets) + candidate_catalog = stac_to_df( + stac_items=candidate_items, + assets=assets, + column_allow_list=allow_list, + column_block_list=block_list, + ) benchmark_items = catalog.search( datetime=times[1], @@ -246,7 +251,12 @@ def test_stac_catalog_comparison_success( bbox=bbox, ).item_collection() - benchmark_catalog = stac_to_df(stac_items=benchmark_items, assets=assets) + benchmark_catalog = stac_to_df( + stac_items=benchmark_items, + assets=assets, + column_allow_list=allow_list, + column_block_list=block_list, + ) arguments = { "candidate_catalog": candidate_catalog, @@ -275,10 +285,12 @@ def test_stac_catalog_comparison_success( @parametrize_with_cases( - "url, collection, time, bbox, assets, exception", + "url, collection, time, bbox, assets, allow_list, block_list, exception", glob="stac_catalog_comparison_fail", ) -def test_stac_catalog_comparison_fail(url, collection, time, bbox, assets, exception): +def test_stac_catalog_comparison_fail( + url, collection, time, bbox, assets, allow_list, block_list, exception +): with raises(exception): catalog = pystac_client.Client.open(url) @@ -288,4 +300,9 @@ def test_stac_catalog_comparison_fail(url, collection, time, bbox, assets, excep bbox=bbox, ).item_collection() - _ = stac_to_df(stac_items=candidate_items, assets=assets) + _ = stac_to_df( + stac_items=candidate_items, + assets=assets, + column_allow_list=allow_list, + column_block_list=block_list, + ) From 57d45f784fff7e5f444984b136f55d7ea6e5e816 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Fri, 22 Mar 2024 13:46:11 -0400 Subject: [PATCH 06/12] Revision requests round 2 --- requirements.txt | 1 - src/gval/utils/loading_datasets.py | 257 +----- tests/cases_catalogs.py | 1158 +++++++++++++++++++++------- tests/cases_stac.py | 187 ----- tests/test_catalogs.py | 12 +- tests/test_stac.py | 105 --- 6 files changed, 938 insertions(+), 782 deletions(-) delete mode 100644 tests/cases_stac.py delete mode 100644 tests/test_stac.py diff --git a/requirements.txt b/requirements.txt index ff5699f7..b0d65de9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,3 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 -stackstac==0.5.0 diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index 1aa4c4cd..cecff924 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -4,7 +4,6 @@ __author__ = "Fernando Aristizabal" -import warnings from typing import Union, Optional, Tuple, Iterable from numbers import Number import ast @@ -13,14 +12,11 @@ import rioxarray as rxr import xarray as xr import numpy as np -from shapely.geometry import MultiPoint, shape from tempfile import NamedTemporaryFile from rio_cogeo.cogeo import cog_translate from rio_cogeo.profiles import cog_profiles from pystac.item_collection import ItemCollection -import stackstac - _MEMORY_STRATEGY = "normal" @@ -240,187 +236,11 @@ def _convert_to_dataset(xr_object=Union[xr.DataArray, xr.Dataset]) -> xr.Dataset return xr_object -def _get_raster_band_nodata(band_metadata, nodata_fill) -> Number: - """ - Extracts nodata information from STAC APIs that implement Raster Extension - - Parameters - ---------- - band_metadata : list - Metadata from raster:bands extension - nodata_fill : Number - Fill in value for missing data - - Returns - ------- - Number - Number representing nodata - - Raises - ------ - ValueError - - """ - - if band_metadata: - prop_string = str(band_metadata.coords["raster:bands"].values) - idx1, idx2 = prop_string.find("{"), prop_string.rfind("}") - - return ast.literal_eval(prop_string[idx1 : idx2 + 1]).get("nodata") - else: - if nodata_fill is None: - raise ValueError( - "Must have nodata fill value if nodata is not present in metadata" - ) - - return nodata_fill - - -def _set_nodata( - stack: xr.DataArray, band_metadata: list = None, nodata_fill: Number = None -) -> Number: - """ - Sets nodata information from STAC APIs that implement Raster Extension - - Parameters - ---------- - stack : xr.DataArray - Data to set nodata attribute - band_metadata : list - Metadata from raster:bands extension - nodata_fill : Number - Fill in value for missing data - - """ - - if stack.rio.nodata is not None: - stack.rio.write_nodata(stack.rio.nodata, inplace=True) - elif stack.rio.encoded_nodata is not None: - stack.rio.write_nodata(stack.rio.encoded_nodata, inplace=True) - else: - stack.rio.write_nodata( - _get_raster_band_nodata(band_metadata, nodata_fill), inplace=True - ) - - -def _set_crs(stack: xr.DataArray, band_metadata: list = None) -> Number: - """ - - Parameters - ---------- - stack : xr.DataArray - Original data with no information - band_metadata : dict - Information with band metadata - - Returns - ------- - Xarray DataArray with proper CRS - - """ - - if stack.rio.crs is not None: - return stack.rio.write_crs(stack.rio.crs) - else: - return stack.rio.write_crs(f"EPSG:{band_metadata['epsg'].values}") - - -def get_stac_data( - stac_items: ItemCollection, - bands: list = None, - time_aggregate: str = None, - resolution: int = None, - nodata_fill: Number = None, -) -> xr.Dataset: - """Transform STAC Items in to an xarray object - - Parameters - ---------- - stac_items : ItemCollection - STAC Item Collection returned from pystac client - bands: list, default = None - Bands to retrieve from service - time_aggregate : str, default = None - Method to aggregate multiple time stamps - resolution : int, default = 10 - Resolution to get data from - nodata_fill : Number, default = None - Value to fill nodata where not present - - Returns - ------- - xr.Dataset - Xarray object with resepective STAC API data - - Raises - ------ - ValueError - A valid aggregate must be used for time ranges - - """ - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - stack = stackstac.stack(stac_items, resolution=resolution) - - # Only get unique time indices in case there are duplicates - _, idxs = np.unique(stack.coords["time"], return_index=True) - stack = stack[idxs] - - # Aggregate if there is more than one time - if stack.coords["time"].shape[0] > 1: - crs = stack.rio.crs - if time_aggregate == "mean": - stack = stack.mean(dim="time") - stack.attrs["time_aggregate"] = "mean" - elif time_aggregate == "min": - stack = stack.min(dim="time") - stack.attrs["time_aggregate"] = "min" - elif time_aggregate == "max": - stack = stack.max(dim="time") - stack.attrs["time_aggregate"] = "max" - else: - raise ValueError("A valid aggregate must be used for time ranges") - - stack.rio.write_crs(crs, inplace=True) - else: - stack = stack[0] - stack.attrs["time_aggregate"] = "none" - - # Select specific bands - if bands is not None: - bands = [bands] if isinstance(bands, str) else bands - stack = stack.sel({"band": bands}) - - band_metadata = ( - stack.coords["raster:bands"] if "raster:bands" in stack.coords else None - ) - if "band" in stack.dims: - og_names = [name for name in stack.coords["band"]] - names = [f"band_{x + 1}" for x in range(len(stack.coords["band"]))] - stack = stack.assign_coords({"band": names}).to_dataset(dim="band") - - for metadata, var, og_var in zip(band_metadata, stack.data_vars, og_names): - _set_nodata(stack[var], metadata, nodata_fill) - stack[var] = _set_crs(stack[var], band_metadata) - stack[var].attrs["original_name"] = og_var - - else: - stack = stack.to_dataset(name="band_1") - _set_nodata(stack["band_1"], band_metadata, nodata_fill) - stack["band_1"] = _set_crs(stack["band_1"]) - stack["band_1"].attrs["original_name"] = ( - bands[0] if isinstance(bands, list) else bands - ) - - return stack - - def stac_to_df( stac_items: ItemCollection, assets: list = None, - column_allow_list: list = None, - column_block_list: list = None, + attribute_allow_list: list = None, + attribute_block_list: list = None, ) -> pd.DataFrame: """Convert STAC Items in to a DataFrame @@ -430,9 +250,9 @@ def stac_to_df( STAC Item Collection returned from pystac client assets : list, default = None Assets to keep, (keep all if None) - column_allow_list: list, default = None + attribute_allow_list: list, default = None List of columns to allow in the result DataFrame - column_block_list: list, default = None + attribute_block_list: list, default = None List of columns to remove in the result DataFrame Returns @@ -449,55 +269,46 @@ def stac_to_df( item_dfs, compare_idx = [], 1 - # Iterate through each STAC Item + # Iterate through each STAC Item and make a unique row for each asset for item in stac_items: item_dict = item.to_dict() - item_columns = {} - - # Get columns for all collection level and item level properties - for key, val in item_dict["properties"].items(): - if not isinstance(val, list): - if isinstance(val, dict): - for k, v in val.items(): - item_columns[k] = [v] - else: - item_columns[key] = [val] - - item_columns["bbox"] = MultiPoint(np.array(item_dict["bbox"]).reshape(2, 2)).wkt - item_columns["geometry"] = shape(item_dict["geometry"]).wkt - - unique_keys = [] - for k, v in item_dict["assets"].items(): - for key in v.keys(): - if key not in unique_keys: - unique_keys.append(key) - - # Create new row for each asset with and assign compare_id and map_id - asset_dfs = [] - for k, v in item_dict["assets"].items(): - if assets is None or k in assets: - asset_columns = item_columns.copy() - - asset_columns[key] = [str(v.get(key, "N/a"))] - asset_columns["compare_id"] = compare_idx - asset_columns["map_id"] = v["href"] + item_df = pd.json_normalize(item_dict) + mask = item_df.columns.str.contains("assets.*") + og_df = item_df.loc[:, ~mask] + + if ( + assets is not None + and np.sum([asset not in item_dict["assets"].keys() for asset in assets]) + > 0 + ): + raise ValueError("Non existent asset in parameter assets") + + dfs = [] + + # Make a unique row for each asset + for key, val in item_dict["assets"].items(): + if assets is None or key in assets: + df = pd.json_normalize(val) + df["asset"] = key + df["compare_id"] = compare_idx + df["map_id"] = val["href"] compare_idx += 1 - asset_columns["asset"] = [k] - for key in unique_keys: - asset_columns[key] = [str(v.get(key, "N/a"))] + concat_df = pd.concat([og_df, df], axis=1) + dfs.append(concat_df.loc[:, ~concat_df.columns.duplicated()]) - asset_dfs.append(pd.DataFrame(asset_columns)) + if len(dfs) < 1: + raise ValueError() - item_dfs.append(pd.concat(asset_dfs)) + item_dfs.append(pd.concat(dfs, ignore_index=True)) # Concatenate the DataFrames and remove unwanted columns if allow and block lists exist catalog_df = pd.concat(item_dfs, ignore_index=True) - if column_allow_list is not None: - catalog_df = catalog_df[column_allow_list] + if attribute_allow_list is not None: + catalog_df = catalog_df[attribute_allow_list] - if column_block_list is not None: - catalog_df = catalog_df.drop(column_block_list, axis=1) + if attribute_block_list is not None: + catalog_df = catalog_df.drop(attribute_block_list, axis=1) return catalog_df diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index 3bc835a2..1bdce159 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -471,215 +471,534 @@ def case_compare_catalogs_fail( expected_stac_df = [ { - "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "platform_candidate": ["sentinel-2b", "sentinel-2b"], - "constellation_candidate": ["sentinel-2", "sentinel-2"], - "eo:cloud_cover_candidate": [26.25798, 26.031335], - "proj:epsg_candidate": [32613, 32613], - "mgrs:utm_zone_candidate": [13, 13], - "mgrs:latitude_band_candidate": ["S", "S"], - "mgrs:grid_square_candidate": ["DV", "DV"], - "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], - "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], - "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], - "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], - "s2:saturated_defective_pixel_percentage_candidate": [0, 0], - "s2:dark_features_percentage_candidate": [2.557362, 4.251513], - "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], - "s2:vegetation_percentage_candidate": [6.046846, 6.331863], - "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], - "s2:water_percentage_candidate": [0.019934, 0.045153], - "s2:unclassified_percentage_candidate": [5.82255, 5.403911], - "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], - "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], - "s2:thin_cirrus_percentage_candidate": [0, 0], - "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], - "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_candidate": ["05.00", "02.14"], - "s2:product_uri_candidate": [ + "type_candidate": ["Feature", "Feature"], + "stac_version_candidate": ["1.0.0", "1.0.0"], + "id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], + "links_candidate": [ + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/S2B_13SDV_20200401_1_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A/thumbnail", + }, + ], + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/S2B_13SDV_20200401_0_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A/thumbnail", + }, + ], + ], + "bbox_candidate": [ + [ + -106.11191385205835, + 35.150822790058335, + -105.53827534157463, + 36.14367977962539, + ], + [ + -106.11191385205835, + 35.1499211736146, + -105.53527335203748, + 36.14369323431527, + ], + ], + "stac_extensions_candidate": [ + [ + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + ], + [ + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + ], + ], + "collection_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], + "properties.created_candidate": [ + "2023-10-07T12:09:07.273Z", + "2022-11-06T10:14:16.681Z", + ], + "properties.platform_candidate": ["sentinel-2b", "sentinel-2b"], + "properties.constellation_candidate": ["sentinel-2", "sentinel-2"], + "properties.instruments_candidate": [["msi"], ["msi"]], + "properties.eo:cloud_cover_candidate": [26.25798, 26.031335], + "properties.proj:epsg_candidate": [32613, 32613], + "properties.mgrs:utm_zone_candidate": [13, 13], + "properties.mgrs:latitude_band_candidate": ["S", "S"], + "properties.mgrs:grid_square_candidate": ["DV", "DV"], + "properties.grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], + "properties.view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], + "properties.view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], + "properties.s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], + "properties.s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], + "properties.s2:saturated_defective_pixel_percentage_candidate": [0, 0], + "properties.s2:dark_features_percentage_candidate": [2.557362, 4.251513], + "properties.s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], + "properties.s2:vegetation_percentage_candidate": [6.046846, 6.331863], + "properties.s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], + "properties.s2:water_percentage_candidate": [0.019934, 0.045153], + "properties.s2:unclassified_percentage_candidate": [5.82255, 5.403911], + "properties.s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], + "properties.s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], + "properties.s2:thin_cirrus_percentage_candidate": [0, 0], + "properties.s2:snow_ice_percentage_candidate": [3.051021, 3.01438], + "properties.s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], + "properties.s2:processing_baseline_candidate": ["05.00", "02.14"], + "properties.s2:product_uri_candidate": [ "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", ], - "s2:generation_time_candidate": [ + "properties.s2:generation_time_candidate": [ "2023-06-24T05:35:45.000000Z", "2020-04-01T22:01:55.000000Z", ], - "s2:datatake_id_candidate": [ + "properties.s2:datatake_id_candidate": [ "GS2B_20200401T174909_016040_N05.00", "GS2B_20200401T174909_016040_N02.14", ], - "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_candidate": [ + "properties.s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], + "properties.s2:datastrip_id_candidate": [ "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", ], - "s2:granule_id_candidate": [ + "properties.s2:granule_id_candidate": [ "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", ], - "s2:reflectance_conversion_factor_candidate": [ + "properties.s2:reflectance_conversion_factor_candidate": [ 1.00356283682453, 1.00356283682453, ], - "datetime_candidate": [ + "properties.datetime_candidate": [ "2020-04-01T18:04:04.327000Z", "2020-04-01T18:04:04.327000Z", ], - "s2:sequence_candidate": ["1", "0"], - "earthsearch:s3_path_candidate": [ + "properties.s2:sequence_candidate": ["1", "0"], + "properties.earthsearch:s3_path_candidate": [ "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", ], - "earthsearch:payload_id_candidate": [ + "properties.earthsearch:payload_id_candidate": [ "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", ], - "earthsearch:boa_offset_applied_candidate": [True, False], - "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], - "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "bbox_candidate": [ - "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", - ], - "geometry_candidate": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], - "compare_id": [1, 2], - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + "properties.earthsearch:boa_offset_applied_candidate": [True, False], + "properties.processing:software.sentinel2-to-stac_candidate": [ + "0.1.1", + "0.1.0", + ], + "properties.updated_candidate": [ + "2023-10-07T12:09:07.273Z", + "2022-11-06T10:14:16.681Z", + ], + "geometry.type_candidate": ["Polygon", "Polygon"], + "geometry.coordinates_candidate": [ + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.0982941692468, 35.150822790058335], + [-105.85393882465051, 35.15385918076215], + [-105.68102037327243, 35.69893282033011], + [-105.59450557216445, 35.97641506053815], + [-105.56226433775963, 36.07584727804726], + [-105.53827534157463, 36.14367977962539], + [-106.11191385205835, 36.13972769324406], + ] + ], + [ + [ + [-106.11191385205835, 36.13972769324406], + [-105.53527335203748, 36.14369323431527], + [-105.56579262097148, 36.05653228802631], + [-105.68980719734964, 35.659112338538634], + [-105.85157080324588, 35.15190642354915], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], ], - "asset_candidate": ["aot", "aot"], "href_candidate": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", ], - "type_candidate": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], "title_candidate": [ "Aerosol optical thickness (AOT)", "Aerosol optical thickness (AOT)", ], - "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], + "proj:shape_candidate": [[5490, 5490], [5490, 5490]], "proj:transform_candidate": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", + [20, 0, 399960, 0, -20, 4000020], + [20, 0, 399960, 0, -20, 4000020], ], "raster:bands_candidate": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - ], - "eo:bands_candidate": ["N/a", "N/a"], - "gsd_candidate": ["N/a", "N/a"], - "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "platform_benchmark": ["sentinel-2a", "sentinel-2a"], - "constellation_benchmark": ["sentinel-2", "sentinel-2"], - "eo:cloud_cover_benchmark": [0.394644, 0.946059], - "proj:epsg_benchmark": [32613, 32613], - "mgrs:utm_zone_benchmark": [13, 13], - "mgrs:latitude_band_benchmark": ["S", "S"], - "mgrs:grid_square_benchmark": ["DV", "DV"], - "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], - "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], - "s2:degraded_msi_data_percentage_benchmark": [0, 0], - "s2:nodata_pixel_percentage_benchmark": [0, 0], - "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], - "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], - "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], - "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], - "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], - "s2:water_percentage_benchmark": [0.095783, 0.093679], - "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], - "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], - "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], - "s2:thin_cirrus_percentage_benchmark": [0, 0], - "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], - "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_benchmark": ["05.00", "02.14"], - "s2:product_uri_benchmark": [ + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + ], + "roles_candidate": [["data", "reflectance"], ["data", "reflectance"]], + "asset_candidate": ["aot", "aot"], + "compare_id": [1, 2], + "map_id_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "type_benchmark": ["Feature", "Feature"], + "stac_version_benchmark": ["1.0.0", "1.0.0"], + "id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], + "links_benchmark": [ + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/S2A_13SDV_20200403_1_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A/thumbnail", + }, + ], + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/S2A_13SDV_20200403_0_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A/thumbnail", + }, + ], + ], + "bbox_benchmark": [ + [ + -106.11191385205835, + 35.1499211736146, + -104.89152152018616, + 36.14484027029347, + ], + [ + -106.11191385205835, + 35.1499211736146, + -104.89152152018616, + 36.14484027029347, + ], + ], + "stac_extensions_benchmark": [ + [ + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + ], + [ + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + ], + ], + "collection_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], + "properties.created_benchmark": [ + "2023-10-08T00:32:51.304Z", + "2022-11-06T07:21:36.990Z", + ], + "properties.platform_benchmark": ["sentinel-2a", "sentinel-2a"], + "properties.constellation_benchmark": ["sentinel-2", "sentinel-2"], + "properties.instruments_benchmark": [["msi"], ["msi"]], + "properties.eo:cloud_cover_benchmark": [0.394644, 0.946059], + "properties.proj:epsg_benchmark": [32613, 32613], + "properties.mgrs:utm_zone_benchmark": [13, 13], + "properties.mgrs:latitude_band_benchmark": ["S", "S"], + "properties.mgrs:grid_square_benchmark": ["DV", "DV"], + "properties.grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], + "properties.view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], + "properties.view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], + "properties.s2:degraded_msi_data_percentage_benchmark": [0, 0], + "properties.s2:nodata_pixel_percentage_benchmark": [0, 0], + "properties.s2:saturated_defective_pixel_percentage_benchmark": [0, 0], + "properties.s2:dark_features_percentage_benchmark": [0.619855, 0.411445], + "properties.s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], + "properties.s2:vegetation_percentage_benchmark": [13.550997, 13.711172], + "properties.s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], + "properties.s2:water_percentage_benchmark": [0.095783, 0.093679], + "properties.s2:unclassified_percentage_benchmark": [0.236433, 2.878225], + "properties.s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], + "properties.s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], + "properties.s2:thin_cirrus_percentage_benchmark": [0, 0], + "properties.s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], + "properties.s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], + "properties.s2:processing_baseline_benchmark": ["05.00", "02.14"], + "properties.s2:product_uri_benchmark": [ "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", ], - "s2:generation_time_benchmark": [ + "properties.s2:generation_time_benchmark": [ "2023-05-10T05:34:45.000000Z", "2020-04-03T22:01:05.000000Z", ], - "s2:datatake_id_benchmark": [ + "properties.s2:datatake_id_benchmark": [ "GS2A_20200403T173901_024977_N05.00", "GS2A_20200403T173901_024977_N02.14", ], - "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_benchmark": [ + "properties.s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], + "properties.s2:datastrip_id_benchmark": [ "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", ], - "s2:granule_id_benchmark": [ + "properties.s2:granule_id_benchmark": [ "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", ], - "s2:reflectance_conversion_factor_benchmark": [ + "properties.s2:reflectance_conversion_factor_benchmark": [ 1.00241535908783, 1.00241535908783, ], - "datetime_benchmark": [ + "properties.datetime_benchmark": [ "2020-04-03T17:54:07.524000Z", "2020-04-03T17:54:07.524000Z", ], - "s2:sequence_benchmark": ["1", "0"], - "earthsearch:s3_path_benchmark": [ + "properties.s2:sequence_benchmark": ["1", "0"], + "properties.earthsearch:s3_path_benchmark": [ "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", ], - "earthsearch:payload_id_benchmark": [ + "properties.earthsearch:payload_id_benchmark": [ "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", ], - "earthsearch:boa_offset_applied_benchmark": [True, False], - "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], - "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "bbox_benchmark": [ - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - ], - "geometry_benchmark": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + "properties.earthsearch:boa_offset_applied_benchmark": [True, False], + "properties.processing:software.sentinel2-to-stac_benchmark": [ + "0.1.1", + "0.1.0", + ], + "properties.updated_benchmark": [ + "2023-10-08T00:32:51.304Z", + "2022-11-06T07:21:36.990Z", + ], + "geometry.type_benchmark": ["Polygon", "Polygon"], + "geometry.coordinates_benchmark": [ + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.09828205302668, 35.1499211736146], + [-104.89285176524281, 35.154851672138626], + [-104.89152152018616, 36.14484027029347], + [-106.11191385205835, 36.13972769324406], + ] + ], + [ + [ + [-106.11191385205835, 36.13972769324406], + [-104.89152152018616, 36.14484027029347], + [-104.89285176524281, 35.154851672138626], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], ], - "asset_benchmark": ["aot", "aot"], "href_benchmark": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", ], - "type_benchmark": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], "title_benchmark": [ "Aerosol optical thickness (AOT)", "Aerosol optical thickness (AOT)", ], - "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], + "proj:shape_benchmark": [[5490, 5490], [5490, 5490]], "proj:transform_benchmark": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", + [20, 0, 399960, 0, -20, 4000020], + [20, 0, 399960, 0, -20, 4000020], ], "raster:bands_benchmark": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + ], + "roles_benchmark": [["data", "reflectance"], ["data", "reflectance"]], + "asset_benchmark": ["aot", "aot"], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", ], - "eo:bands_benchmark": ["N/a", "N/a"], - "gsd_benchmark": ["N/a", "N/a"], "band": ["1", "1"], "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], "mean_absolute_error": [25.393386840820312, 11.947012901306152], @@ -701,213 +1020,532 @@ def case_compare_catalogs_fail( "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], }, { - "created_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "platform_candidate": ["sentinel-2b", "sentinel-2b"], - "constellation_candidate": ["sentinel-2", "sentinel-2"], - "proj:epsg_candidate": [32613, 32613], - "mgrs:utm_zone_candidate": [13, 13], - "mgrs:latitude_band_candidate": ["S", "S"], - "mgrs:grid_square_candidate": ["DV", "DV"], - "grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], - "view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], - "s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], - "s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], - "s2:saturated_defective_pixel_percentage_candidate": [0, 0], - "s2:dark_features_percentage_candidate": [2.557362, 4.251513], - "s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], - "s2:vegetation_percentage_candidate": [6.046846, 6.331863], - "s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], - "s2:water_percentage_candidate": [0.019934, 0.045153], - "s2:unclassified_percentage_candidate": [5.82255, 5.403911], - "s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], - "s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], - "s2:thin_cirrus_percentage_candidate": [0, 0], - "s2:snow_ice_percentage_candidate": [3.051021, 3.01438], - "s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_candidate": ["05.00", "02.14"], - "s2:product_uri_candidate": [ + "type_candidate": ["Feature", "Feature"], + "stac_version_candidate": ["1.0.0", "1.0.0"], + "id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], + "links_candidate": [ + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/S2B_13SDV_20200401_1_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A/thumbnail", + }, + ], + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/S2B_13SDV_20200401_0_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A/thumbnail", + }, + ], + ], + "bbox_candidate": [ + [ + -106.11191385205835, + 35.150822790058335, + -105.53827534157463, + 36.14367977962539, + ], + [ + -106.11191385205835, + 35.1499211736146, + -105.53527335203748, + 36.14369323431527, + ], + ], + "stac_extensions_candidate": [ + [ + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + ], + [ + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + ], + ], + "collection_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], + "properties.created_candidate": [ + "2023-10-07T12:09:07.273Z", + "2022-11-06T10:14:16.681Z", + ], + "properties.platform_candidate": ["sentinel-2b", "sentinel-2b"], + "properties.constellation_candidate": ["sentinel-2", "sentinel-2"], + "properties.instruments_candidate": [["msi"], ["msi"]], + "properties.eo:cloud_cover_candidate": [26.25798, 26.031335], + "properties.proj:epsg_candidate": [32613, 32613], + "properties.mgrs:latitude_band_candidate": ["S", "S"], + "properties.mgrs:grid_square_candidate": ["DV", "DV"], + "properties.grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], + "properties.view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], + "properties.view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], + "properties.s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], + "properties.s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], + "properties.s2:saturated_defective_pixel_percentage_candidate": [0, 0], + "properties.s2:dark_features_percentage_candidate": [2.557362, 4.251513], + "properties.s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], + "properties.s2:vegetation_percentage_candidate": [6.046846, 6.331863], + "properties.s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], + "properties.s2:water_percentage_candidate": [0.019934, 0.045153], + "properties.s2:unclassified_percentage_candidate": [5.82255, 5.403911], + "properties.s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], + "properties.s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], + "properties.s2:thin_cirrus_percentage_candidate": [0, 0], + "properties.s2:snow_ice_percentage_candidate": [3.051021, 3.01438], + "properties.s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], + "properties.s2:processing_baseline_candidate": ["05.00", "02.14"], + "properties.s2:product_uri_candidate": [ "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", ], - "s2:generation_time_candidate": [ + "properties.s2:generation_time_candidate": [ "2023-06-24T05:35:45.000000Z", "2020-04-01T22:01:55.000000Z", ], - "s2:datatake_id_candidate": [ + "properties.s2:datatake_id_candidate": [ "GS2B_20200401T174909_016040_N05.00", "GS2B_20200401T174909_016040_N02.14", ], - "s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_candidate": [ + "properties.s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], + "properties.s2:datastrip_id_candidate": [ "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", ], - "s2:granule_id_candidate": [ + "properties.s2:granule_id_candidate": [ "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", ], - "s2:reflectance_conversion_factor_candidate": [ + "properties.s2:reflectance_conversion_factor_candidate": [ 1.00356283682453, 1.00356283682453, ], - "datetime_candidate": [ + "properties.datetime_candidate": [ "2020-04-01T18:04:04.327000Z", "2020-04-01T18:04:04.327000Z", ], - "s2:sequence_candidate": ["1", "0"], - "earthsearch:s3_path_candidate": [ + "properties.s2:sequence_candidate": ["1", "0"], + "properties.earthsearch:s3_path_candidate": [ "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", ], - "earthsearch:payload_id_candidate": [ + "properties.earthsearch:payload_id_candidate": [ "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", ], - "earthsearch:boa_offset_applied_candidate": [True, False], - "sentinel2-to-stac_candidate": ["0.1.1", "0.1.0"], - "updated_candidate": ["2023-10-07T12:09:07.273Z", "2022-11-06T10:14:16.681Z"], - "bbox_candidate": [ - "MULTIPOINT (-106.11191385205835 35.150822790058335, -105.53827534157463 36.14367977962539)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -105.53527335203748 36.14369323431527)", - ], - "geometry_candidate": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.0982941692468 35.150822790058335, -105.85393882465051 35.15385918076215, -105.68102037327243 35.69893282033011, -105.59450557216445 35.97641506053815, -105.56226433775963 36.07584727804726, -105.53827534157463 36.14367977962539, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -105.53527335203748 36.14369323431527, -105.56579262097148 36.05653228802631, -105.68980719734964 35.659112338538634, -105.85157080324588 35.15190642354915, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_candidate": ["['data', 'reflectance']", "['data', 'reflectance']"], - "compare_id": [1, 2], - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + "properties.earthsearch:boa_offset_applied_candidate": [True, False], + "properties.processing:software.sentinel2-to-stac_candidate": [ + "0.1.1", + "0.1.0", + ], + "properties.updated_candidate": [ + "2023-10-07T12:09:07.273Z", + "2022-11-06T10:14:16.681Z", + ], + "geometry.type_candidate": ["Polygon", "Polygon"], + "geometry.coordinates_candidate": [ + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.0982941692468, 35.150822790058335], + [-105.85393882465051, 35.15385918076215], + [-105.68102037327243, 35.69893282033011], + [-105.59450557216445, 35.97641506053815], + [-105.56226433775963, 36.07584727804726], + [-105.53827534157463, 36.14367977962539], + [-106.11191385205835, 36.13972769324406], + ] + ], + [ + [ + [-106.11191385205835, 36.13972769324406], + [-105.53527335203748, 36.14369323431527], + [-105.56579262097148, 36.05653228802631], + [-105.68980719734964, 35.659112338538634], + [-105.85157080324588, 35.15190642354915], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], ], - "asset_candidate": ["aot", "aot"], "href_candidate": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", ], - "type_candidate": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], "title_candidate": [ "Aerosol optical thickness (AOT)", "Aerosol optical thickness (AOT)", ], - "proj:shape_candidate": ["[5490, 5490]", "[5490, 5490]"], + "proj:shape_candidate": [[5490, 5490], [5490, 5490]], "proj:transform_candidate": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", + [20, 0, 399960, 0, -20, 4000020], + [20, 0, 399960, 0, -20, 4000020], ], "raster:bands_candidate": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - ], - "eo:bands_candidate": ["N/a", "N/a"], - "gsd_candidate": ["N/a", "N/a"], - "created_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "platform_benchmark": ["sentinel-2a", "sentinel-2a"], - "constellation_benchmark": ["sentinel-2", "sentinel-2"], - "proj:epsg_benchmark": [32613, 32613], - "mgrs:utm_zone_benchmark": [13, 13], - "mgrs:latitude_band_benchmark": ["S", "S"], - "mgrs:grid_square_benchmark": ["DV", "DV"], - "grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], - "view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], - "view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], - "s2:degraded_msi_data_percentage_benchmark": [0, 0], - "s2:nodata_pixel_percentage_benchmark": [0, 0], - "s2:saturated_defective_pixel_percentage_benchmark": [0, 0], - "s2:dark_features_percentage_benchmark": [0.619855, 0.411445], - "s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], - "s2:vegetation_percentage_benchmark": [13.550997, 13.711172], - "s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], - "s2:water_percentage_benchmark": [0.095783, 0.093679], - "s2:unclassified_percentage_benchmark": [0.236433, 2.878225], - "s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], - "s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], - "s2:thin_cirrus_percentage_benchmark": [0, 0], - "s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], - "s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], - "s2:processing_baseline_benchmark": ["05.00", "02.14"], - "s2:product_uri_benchmark": [ + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + ], + "roles_candidate": [["data", "reflectance"], ["data", "reflectance"]], + "asset_candidate": ["aot", "aot"], + "compare_id": [1, 2], + "map_id_candidate": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", + ], + "type_benchmark": ["Feature", "Feature"], + "stac_version_benchmark": ["1.0.0", "1.0.0"], + "id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], + "links_benchmark": [ + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/S2A_13SDV_20200403_1_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A/thumbnail", + }, + ], + [ + { + "rel": "self", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A", + "type": "application/geo+json", + }, + { + "rel": "canonical", + "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/S2A_13SDV_20200403_0_L2A.json", + "type": "application/json", + }, + { + "rel": "license", + "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", + }, + { + "rel": "derived_from", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", + "type": "application/geo+json", + }, + { + "rel": "parent", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "collection", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", + "type": "application/json", + }, + { + "rel": "root", + "href": "https://earth-search.aws.element84.com/v1", + "type": "application/json", + "title": "Earth Search by Element 84", + }, + { + "rel": "thumbnail", + "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A/thumbnail", + }, + ], + ], + "bbox_benchmark": [ + [ + -106.11191385205835, + 35.1499211736146, + -104.89152152018616, + 36.14484027029347, + ], + [ + -106.11191385205835, + 35.1499211736146, + -104.89152152018616, + 36.14484027029347, + ], + ], + "stac_extensions_benchmark": [ + [ + "https://stac-extensions.github.io/projection/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.1.0/schema.json", + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + ], + [ + "https://stac-extensions.github.io/grid/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json", + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.1.0/schema.json", + "https://stac-extensions.github.io/processing/v1.1.0/schema.json", + "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", + ], + ], + "collection_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], + "properties.created_benchmark": [ + "2023-10-08T00:32:51.304Z", + "2022-11-06T07:21:36.990Z", + ], + "properties.platform_benchmark": ["sentinel-2a", "sentinel-2a"], + "properties.constellation_benchmark": ["sentinel-2", "sentinel-2"], + "properties.instruments_benchmark": [["msi"], ["msi"]], + "properties.eo:cloud_cover_benchmark": [0.394644, 0.946059], + "properties.proj:epsg_benchmark": [32613, 32613], + "properties.mgrs:latitude_band_benchmark": ["S", "S"], + "properties.mgrs:grid_square_benchmark": ["DV", "DV"], + "properties.grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], + "properties.view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], + "properties.view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], + "properties.s2:degraded_msi_data_percentage_benchmark": [0, 0], + "properties.s2:nodata_pixel_percentage_benchmark": [0, 0], + "properties.s2:saturated_defective_pixel_percentage_benchmark": [0, 0], + "properties.s2:dark_features_percentage_benchmark": [0.619855, 0.411445], + "properties.s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], + "properties.s2:vegetation_percentage_benchmark": [13.550997, 13.711172], + "properties.s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], + "properties.s2:water_percentage_benchmark": [0.095783, 0.093679], + "properties.s2:unclassified_percentage_benchmark": [0.236433, 2.878225], + "properties.s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], + "properties.s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], + "properties.s2:thin_cirrus_percentage_benchmark": [0, 0], + "properties.s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], + "properties.s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], + "properties.s2:processing_baseline_benchmark": ["05.00", "02.14"], + "properties.s2:product_uri_benchmark": [ "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", ], - "s2:generation_time_benchmark": [ + "properties.s2:generation_time_benchmark": [ "2023-05-10T05:34:45.000000Z", "2020-04-03T22:01:05.000000Z", ], - "s2:datatake_id_benchmark": [ + "properties.s2:datatake_id_benchmark": [ "GS2A_20200403T173901_024977_N05.00", "GS2A_20200403T173901_024977_N02.14", ], - "s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], - "s2:datastrip_id_benchmark": [ + "properties.s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], + "properties.s2:datastrip_id_benchmark": [ "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", ], - "s2:granule_id_benchmark": [ + "properties.s2:granule_id_benchmark": [ "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", ], - "s2:reflectance_conversion_factor_benchmark": [ + "properties.s2:reflectance_conversion_factor_benchmark": [ 1.00241535908783, 1.00241535908783, ], - "datetime_benchmark": [ + "properties.datetime_benchmark": [ "2020-04-03T17:54:07.524000Z", "2020-04-03T17:54:07.524000Z", ], - "s2:sequence_benchmark": ["1", "0"], - "earthsearch:s3_path_benchmark": [ + "properties.s2:sequence_benchmark": ["1", "0"], + "properties.earthsearch:s3_path_benchmark": [ "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", ], - "earthsearch:payload_id_benchmark": [ + "properties.earthsearch:payload_id_benchmark": [ "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", ], - "earthsearch:boa_offset_applied_benchmark": [True, False], - "sentinel2-to-stac_benchmark": ["0.1.1", "0.1.0"], - "updated_benchmark": ["2023-10-08T00:32:51.304Z", "2022-11-06T07:21:36.990Z"], - "bbox_benchmark": [ - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - "MULTIPOINT (-106.11191385205835 35.1499211736146, -104.89152152018616 36.14484027029347)", - ], - "geometry_benchmark": [ - "POLYGON ((-106.11191385205835 36.13972769324406, -106.09828205302668 35.1499211736146, -104.89285176524281 35.154851672138626, -104.89152152018616 36.14484027029347, -106.11191385205835 36.13972769324406))", - "POLYGON ((-106.11191385205835 36.13972769324406, -104.89152152018616 36.14484027029347, -104.89285176524281 35.154851672138626, -106.09828205302668 35.1499211736146, -106.11191385205835 36.13972769324406))", - ], - "roles_benchmark": ["['data', 'reflectance']", "['data', 'reflectance']"], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", + "properties.earthsearch:boa_offset_applied_benchmark": [True, False], + "properties.processing:software.sentinel2-to-stac_benchmark": [ + "0.1.1", + "0.1.0", + ], + "properties.updated_benchmark": [ + "2023-10-08T00:32:51.304Z", + "2022-11-06T07:21:36.990Z", + ], + "geometry.type_benchmark": ["Polygon", "Polygon"], + "geometry.coordinates_benchmark": [ + [ + [ + [-106.11191385205835, 36.13972769324406], + [-106.09828205302668, 35.1499211736146], + [-104.89285176524281, 35.154851672138626], + [-104.89152152018616, 36.14484027029347], + [-106.11191385205835, 36.13972769324406], + ] + ], + [ + [ + [-106.11191385205835, 36.13972769324406], + [-104.89152152018616, 36.14484027029347], + [-104.89285176524281, 35.154851672138626], + [-106.09828205302668, 35.1499211736146], + [-106.11191385205835, 36.13972769324406], + ] + ], ], - "asset_benchmark": ["aot", "aot"], "href_benchmark": [ "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", ], - "type_benchmark": [ - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; application=geotiff; profile=cloud-optimized", - ], "title_benchmark": [ "Aerosol optical thickness (AOT)", "Aerosol optical thickness (AOT)", ], - "proj:shape_benchmark": ["[5490, 5490]", "[5490, 5490]"], + "proj:shape_benchmark": [[5490, 5490], [5490, 5490]], "proj:transform_benchmark": [ - "[20, 0, 399960, 0, -20, 4000020]", - "[20, 0, 399960, 0, -20, 4000020]", + [20, 0, 399960, 0, -20, 4000020], + [20, 0, 399960, 0, -20, 4000020], ], "raster:bands_benchmark": [ - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", - "[{'nodata': 0, 'data_type': 'uint16', 'bits_per_sample': 15, 'spatial_resolution': 20, 'scale': 0.001, 'offset': 0}]", + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + [ + { + "nodata": 0, + "data_type": "uint16", + "bits_per_sample": 15, + "spatial_resolution": 20, + "scale": 0.001, + "offset": 0, + } + ], + ], + "roles_benchmark": [["data", "reflectance"], ["data", "reflectance"]], + "asset_benchmark": ["aot", "aot"], + "map_id_benchmark": [ + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", + "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", ], - "eo:bands_benchmark": ["N/a", "N/a"], - "gsd_benchmark": ["N/a", "N/a"], "band": ["1", "1"], "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], "mean_absolute_error": [25.393386840820312, 11.947012901306152], @@ -916,7 +1554,7 @@ def case_compare_catalogs_fail( ] allow_list = [None, ["map_id", "compare_id"], None] -block_list = [None, None, ["eo:cloud_cover"]] +block_list = [None, None, ["properties.mgrs:utm_zone"]] @parametrize( @@ -950,7 +1588,7 @@ def case_stac_catalog_comparison_success( bad_times = ["2020-04-01"] -bad_assets = ["surface_water", None, None] +bad_assets = [["surface_water"], None, None] exceptions = [ValueError, KeyError, KeyError] bad_allow_list = [None, ["arb"], None] bad_block_list = [None, None, ["arb"]] diff --git a/tests/cases_stac.py b/tests/cases_stac.py deleted file mode 100644 index 7335f91d..00000000 --- a/tests/cases_stac.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -Test functionality for gval/statistics modules -""" - -# __all__ = ['*'] - - -import numpy as np -import pandas as pd -from pytest_cases import parametrize - -url = "https://earth-search.aws.element84.com/v1" -collection = "sentinel-2-l2a" -bbox = [-105.78, 35.79, -105.72, 35.84] - - -bands = ["aot", ["aot"], ["aot"], ["aot"], ["aot", "rededge1"]] - - -time = [ - ["2020-04-01", "2020-04-03"], - ["2020-04-01/2020-04-03", "2020-04-06/2020-04-08"], - ["2020-04-01/2020-04-03", "2020-04-06/2020-04-08"], - ["2020-04-01/2020-04-03", "2020-04-06/2020-04-08"], - ["2020-04-01", "2020-04-03"], -] - -time_aggreagte = [None, "mean", "min", "max", None] - -nodata_fill = [None, None, None, None, 0] - -expected_df = [ - { - "band": ["1"], - "coefficient_of_determination": [-1.4498682989135663], - "mean_absolute_error": [0.025393391237534115], - "mean_absolute_percentage_error": [0.20440844788169352], - "mean_normalized_mean_absolute_error": [0.20592052600829783], - "mean_normalized_root_mean_squared_error": [0.2110757427848459], - "mean_percentage_error": [-0.20540028455073725], - "mean_signed_error": [-0.025329236900295846], - "mean_squared_error": [0.0006775147935436701], - "range_normalized_mean_absolute_error": [0.31741739046917644], - "range_normalized_root_mean_squared_error": [0.32536392930255564], - "root_mean_squared_error": [0.026029114344204452], - "symmetric_mean_absolute_percentage_error": [0.22778286629345662], - }, - { - "band": ["1"], - "coefficient_of_determination": [0.5954048574471591], - "mean_absolute_error": [0.011375152039973328], - "mean_absolute_percentage_error": [0.09299193429857326], - "mean_normalized_mean_absolute_error": [0.09198102377977661], - "mean_normalized_root_mean_squared_error": [0.0950603218421001], - "mean_percentage_error": [-0.03739148233378963], - "mean_signed_error": [-0.004624147232424582], - "mean_squared_error": [0.0001382026920448174], - "range_normalized_mean_absolute_error": [0.10036898858799995], - "range_normalized_root_mean_squared_error": [0.10372909504665788], - "root_mean_squared_error": [0.011755964105287894], - "symmetric_mean_absolute_percentage_error": [0.09373343991103603], - }, - { - "band": ["1"], - "coefficient_of_determination": [0.62257119465569], - "mean_absolute_error": [0.017518999671533943], - "mean_absolute_percentage_error": [0.19568589409159515], - "mean_normalized_mean_absolute_error": [0.15469671510079602], - "mean_normalized_root_mean_squared_error": [0.1715846316993916], - "mean_percentage_error": [0.013368590460465646], - "mean_signed_error": [0.0015139580160649765], - "mean_squared_error": [0.0003775836662784796], - "range_normalized_mean_absolute_error": [0.11450326582701924], - "range_normalized_root_mean_squared_error": [0.12700334769555513], - "root_mean_squared_error": [0.019431512197419933], - "symmetric_mean_absolute_percentage_error": [0.15366954251075923], - }, - { - "band": ["1"], - "coefficient_of_determination": [-0.41271964773267755], - "mean_absolute_error": [0.011908415300546453], - "mean_absolute_percentage_error": [0.0882205870282268], - "mean_normalized_mean_absolute_error": [0.08814209475433238], - "mean_normalized_root_mean_squared_error": [0.08996232968540896], - "mean_percentage_error": [-0.08717281243113986], - "mean_signed_error": [-0.011777460658723765], - "mean_squared_error": [0.00014772792439308438], - "range_normalized_mean_absolute_error": [0.16772415916262606], - "range_normalized_root_mean_squared_error": [0.17118785462101266], - "root_mean_squared_error": [0.0121543376780919], - "symmetric_mean_absolute_percentage_error": [0.09215897319648417], - }, # One more for a multi band example - { - "band": ["1", "2"], - "coefficient_of_determination": [-1.4498682989135663, -0.7514062048150743], - "mean_absolute_error": [0.025393391237534115, 0.10266655092378396], - "mean_absolute_percentage_error": [0.20440844788169352, np.inf], - "mean_normalized_mean_absolute_error": [ - 0.20592052600829783, - 1.5904259451440834, - ], - "mean_normalized_root_mean_squared_error": [ - 0.2110757427848459, - 2.8406238608292003, - ], - "mean_percentage_error": [-0.20540028455073725, 0.8574267097258231], - "mean_signed_error": [-0.025329236900295846, 0.05534935042165941], - "mean_squared_error": [0.0006775147935436701, 0.03362470648587737], - "range_normalized_mean_absolute_error": [ - 0.31741739046917644, - 0.06341747539921179, - ], - "range_normalized_root_mean_squared_error": [ - 0.32536392930255564, - 0.11326852052594609, - ], - "root_mean_squared_error": [0.026029114344204452, 0.18337040787945413], - "symmetric_mean_absolute_percentage_error": [ - 0.22778286629345662, - 0.8343404150271029, - ], - }, -] - - -@parametrize( - "url, collection, bbox, time, bands, time_aggregate, nodata_fill, expected_df", - list( - zip( - [url] * len(time), - [collection] * len(time), - [bbox] * len(time), - time, - bands, - time_aggreagte, - nodata_fill, - expected_df, - ) - ), -) -def case_stac_api_call( - url, collection, bbox, time, bands, time_aggregate, nodata_fill, expected_df -): - return ( - url, - collection, - bbox, - time, - bands, - time_aggregate, - nodata_fill, - pd.DataFrame(expected_df), - ) - - -bands_fail = [["aot"]] - -time_fail = [ - ["2020-04-01/2020-04-03", "2020-04-06/2020-04-08"], -] - -time_aggregate_fail = [None] - -nodata_fill_fail = [None] - -exceptions = [ValueError] - - -@parametrize( - "url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception", - list( - zip( - [url] * len(time_fail), - [collection] * len(time_fail), - [bbox] * len(time_fail), - time_fail, - bands_fail, - time_aggregate_fail, - nodata_fill_fail, - exceptions, - ) - ), -) -def case_stac_api_call_fail( - url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception -): - return url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception diff --git a/tests/test_catalogs.py b/tests/test_catalogs.py index 5392b052..3c5fdbb8 100644 --- a/tests/test_catalogs.py +++ b/tests/test_catalogs.py @@ -241,8 +241,8 @@ def test_stac_catalog_comparison_success( candidate_catalog = stac_to_df( stac_items=candidate_items, assets=assets, - column_allow_list=allow_list, - column_block_list=block_list, + attribute_allow_list=allow_list, + attribute_block_list=block_list, ) benchmark_items = catalog.search( @@ -254,8 +254,8 @@ def test_stac_catalog_comparison_success( benchmark_catalog = stac_to_df( stac_items=benchmark_items, assets=assets, - column_allow_list=allow_list, - column_block_list=block_list, + attribute_allow_list=allow_list, + attribute_block_list=block_list, ) arguments = { @@ -303,6 +303,6 @@ def test_stac_catalog_comparison_fail( _ = stac_to_df( stac_items=candidate_items, assets=assets, - column_allow_list=allow_list, - column_block_list=block_list, + attribute_allow_list=allow_list, + attribute_block_list=block_list, ) diff --git a/tests/test_stac.py b/tests/test_stac.py deleted file mode 100644 index aa0d0906..00000000 --- a/tests/test_stac.py +++ /dev/null @@ -1,105 +0,0 @@ -from pytest_cases import parametrize_with_cases -from pytest import raises -import xarray as xr -import pandas as pd -import pystac_client - -from gval.utils.loading_datasets import get_stac_data - - -@parametrize_with_cases( - "url, collection, bbox, time, bands, time_aggregate, nodata_fill, expected_df", - glob="stac_api_call", -) -def test_stac_api_call( - url, collection, bbox, time, bands, time_aggregate, nodata_fill, expected_df -): - """ - Tests call for stac API, (IDK if this data can be mocked, API calls in unit tests are dubious) - """ - - catalog = pystac_client.Client.open(url) - - candidate_items = catalog.search( - datetime=time[0], - collections=[collection], - bbox=bbox, - ).item_collection() - - candidate = get_stac_data( - stac_items=candidate_items, - bands=bands, - time_aggregate=time_aggregate, - nodata_fill=nodata_fill, - ) - - benchmark_items = catalog.search( - datetime=time[1], - collections=[collection], - bbox=bbox, - ).item_collection() - - benchmark = get_stac_data( - stac_items=benchmark_items, - bands=bands, - time_aggregate=time_aggregate, - nodata_fill=nodata_fill, - ) - - # Registered metrics from previous tests affect this comparison so metrics are explicit - agreement, metrics = candidate.gval.continuous_compare( - benchmark, - metrics=[ - "coefficient_of_determination", - "mean_absolute_error", - "mean_absolute_percentage_error", - "mean_normalized_mean_absolute_error", - "mean_normalized_root_mean_squared_error", - "mean_percentage_error", - "mean_signed_error", - "mean_squared_error", - "range_normalized_mean_absolute_error", - "range_normalized_root_mean_squared_error", - "root_mean_squared_error", - "symmetric_mean_absolute_percentage_error", - ], - ) - - bnds = [bands] if isinstance(bands, str) else bands - - assert isinstance(agreement, xr.Dataset) - assert bnds == [ - agreement[var].attrs["original_name"] for var in agreement.data_vars - ] - - pd.testing.assert_frame_equal( - metrics, expected_df, check_dtype=False - ), "Compute statistics did not return expected values" - - -@parametrize_with_cases( - "url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception", - glob="stac_api_call_fail", -) -def test_stac_api_call_fail( - url, collection, bbox, time, bands, time_aggregate, nodata_fill, exception -): - """ - Tests call for stac API fail - """ - - with raises(exception): - catalog = pystac_client.Client.open(url) - - candidate_items = catalog.search( - datetime=time[0], - collections=[collection], - bbox=bbox, - ).item_collection() - - _ = get_stac_data( - stac_items=candidate_items, - bands=bands, - time_aggregate=time_aggregate, - nodata_fill=nodata_fill, - ) From 79d75446ceccc6aaa43f959683c45b036071eb7a Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 17:55:02 -0400 Subject: [PATCH 07/12] Adjust tests and revisions --- src/gval/utils/loading_datasets.py | 25 +- tests/cases_catalogs.py | 1102 +--------------------------- 2 files changed, 36 insertions(+), 1091 deletions(-) diff --git a/src/gval/utils/loading_datasets.py b/src/gval/utils/loading_datasets.py index cecff924..dd73b87f 100644 --- a/src/gval/utils/loading_datasets.py +++ b/src/gval/utils/loading_datasets.py @@ -7,6 +7,7 @@ from typing import Union, Optional, Tuple, Iterable from numbers import Number import ast +from collections import Counter import pandas as pd import rioxarray as rxr @@ -262,13 +263,32 @@ def stac_to_df( Raises ------ + ValueError + Allow and block lists should be mutually exclusive ValueError No entries in DataFrame due to nonexistent asset + ValueError + There are no assets in this query to run a catalog comparison """ item_dfs, compare_idx = [], 1 + # Check for mutually exclusive lists + if ( + len( + list( + ( + Counter(attribute_allow_list) & Counter(attribute_block_list) + ).elements() + ) + ) + > 0 + ): + raise ValueError( + "There are no assets in this query to run a catalog comparison" + ) + # Iterate through each STAC Item and make a unique row for each asset for item in stac_items: item_dict = item.to_dict() @@ -297,7 +317,10 @@ def stac_to_df( dfs.append(concat_df.loc[:, ~concat_df.columns.duplicated()]) if len(dfs) < 1: - raise ValueError() + raise ValueError( + "There are no assets in this query to run a catalog comparison. " + "Please revisit original query." + ) item_dfs.append(pd.concat(dfs, ignore_index=True)) diff --git a/tests/cases_catalogs.py b/tests/cases_catalogs.py index 1bdce159..1848bad3 100644 --- a/tests/cases_catalogs.py +++ b/tests/cases_catalogs.py @@ -470,1087 +470,9 @@ def case_compare_catalogs_fail( assets = ["aot"] expected_stac_df = [ - { - "type_candidate": ["Feature", "Feature"], - "stac_version_candidate": ["1.0.0", "1.0.0"], - "id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], - "links_candidate": [ - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/S2B_13SDV_20200401_1_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A/thumbnail", - }, - ], - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/S2B_13SDV_20200401_0_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A/thumbnail", - }, - ], - ], - "bbox_candidate": [ - [ - -106.11191385205835, - 35.150822790058335, - -105.53827534157463, - 36.14367977962539, - ], - [ - -106.11191385205835, - 35.1499211736146, - -105.53527335203748, - 36.14369323431527, - ], - ], - "stac_extensions_candidate": [ - [ - "https://stac-extensions.github.io/projection/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/eo/v1.1.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - ], - [ - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/eo/v1.0.0/schema.json", - "https://stac-extensions.github.io/projection/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - ], - ], - "collection_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], - "properties.created_candidate": [ - "2023-10-07T12:09:07.273Z", - "2022-11-06T10:14:16.681Z", - ], - "properties.platform_candidate": ["sentinel-2b", "sentinel-2b"], - "properties.constellation_candidate": ["sentinel-2", "sentinel-2"], - "properties.instruments_candidate": [["msi"], ["msi"]], - "properties.eo:cloud_cover_candidate": [26.25798, 26.031335], - "properties.proj:epsg_candidate": [32613, 32613], - "properties.mgrs:utm_zone_candidate": [13, 13], - "properties.mgrs:latitude_band_candidate": ["S", "S"], - "properties.mgrs:grid_square_candidate": ["DV", "DV"], - "properties.grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], - "properties.view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], - "properties.view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], - "properties.s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], - "properties.s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], - "properties.s2:saturated_defective_pixel_percentage_candidate": [0, 0], - "properties.s2:dark_features_percentage_candidate": [2.557362, 4.251513], - "properties.s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], - "properties.s2:vegetation_percentage_candidate": [6.046846, 6.331863], - "properties.s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], - "properties.s2:water_percentage_candidate": [0.019934, 0.045153], - "properties.s2:unclassified_percentage_candidate": [5.82255, 5.403911], - "properties.s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], - "properties.s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], - "properties.s2:thin_cirrus_percentage_candidate": [0, 0], - "properties.s2:snow_ice_percentage_candidate": [3.051021, 3.01438], - "properties.s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], - "properties.s2:processing_baseline_candidate": ["05.00", "02.14"], - "properties.s2:product_uri_candidate": [ - "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", - "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", - ], - "properties.s2:generation_time_candidate": [ - "2023-06-24T05:35:45.000000Z", - "2020-04-01T22:01:55.000000Z", - ], - "properties.s2:datatake_id_candidate": [ - "GS2B_20200401T174909_016040_N05.00", - "GS2B_20200401T174909_016040_N02.14", - ], - "properties.s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], - "properties.s2:datastrip_id_candidate": [ - "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", - "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", - ], - "properties.s2:granule_id_candidate": [ - "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", - "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", - ], - "properties.s2:reflectance_conversion_factor_candidate": [ - 1.00356283682453, - 1.00356283682453, - ], - "properties.datetime_candidate": [ - "2020-04-01T18:04:04.327000Z", - "2020-04-01T18:04:04.327000Z", - ], - "properties.s2:sequence_candidate": ["1", "0"], - "properties.earthsearch:s3_path_candidate": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", - ], - "properties.earthsearch:payload_id_candidate": [ - "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", - "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", - ], - "properties.earthsearch:boa_offset_applied_candidate": [True, False], - "properties.processing:software.sentinel2-to-stac_candidate": [ - "0.1.1", - "0.1.0", - ], - "properties.updated_candidate": [ - "2023-10-07T12:09:07.273Z", - "2022-11-06T10:14:16.681Z", - ], - "geometry.type_candidate": ["Polygon", "Polygon"], - "geometry.coordinates_candidate": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.0982941692468, 35.150822790058335], - [-105.85393882465051, 35.15385918076215], - [-105.68102037327243, 35.69893282033011], - [-105.59450557216445, 35.97641506053815], - [-105.56226433775963, 36.07584727804726], - [-105.53827534157463, 36.14367977962539], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-105.53527335203748, 36.14369323431527], - [-105.56579262097148, 36.05653228802631], - [-105.68980719734964, 35.659112338538634], - [-105.85157080324588, 35.15190642354915], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "href_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "title_candidate": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_candidate": [[5490, 5490], [5490, 5490]], - "proj:transform_candidate": [ - [20, 0, 399960, 0, -20, 4000020], - [20, 0, 399960, 0, -20, 4000020], - ], - "raster:bands_candidate": [ - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - ], - "roles_candidate": [["data", "reflectance"], ["data", "reflectance"]], - "asset_candidate": ["aot", "aot"], - "compare_id": [1, 2], - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "type_benchmark": ["Feature", "Feature"], - "stac_version_benchmark": ["1.0.0", "1.0.0"], - "id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], - "links_benchmark": [ - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/S2A_13SDV_20200403_1_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A/thumbnail", - }, - ], - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/S2A_13SDV_20200403_0_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A/thumbnail", - }, - ], - ], - "bbox_benchmark": [ - [ - -106.11191385205835, - 35.1499211736146, - -104.89152152018616, - 36.14484027029347, - ], - [ - -106.11191385205835, - 35.1499211736146, - -104.89152152018616, - 36.14484027029347, - ], - ], - "stac_extensions_benchmark": [ - [ - "https://stac-extensions.github.io/projection/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/eo/v1.1.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - ], - [ - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/eo/v1.0.0/schema.json", - "https://stac-extensions.github.io/projection/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - ], - ], - "collection_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], - "properties.created_benchmark": [ - "2023-10-08T00:32:51.304Z", - "2022-11-06T07:21:36.990Z", - ], - "properties.platform_benchmark": ["sentinel-2a", "sentinel-2a"], - "properties.constellation_benchmark": ["sentinel-2", "sentinel-2"], - "properties.instruments_benchmark": [["msi"], ["msi"]], - "properties.eo:cloud_cover_benchmark": [0.394644, 0.946059], - "properties.proj:epsg_benchmark": [32613, 32613], - "properties.mgrs:utm_zone_benchmark": [13, 13], - "properties.mgrs:latitude_band_benchmark": ["S", "S"], - "properties.mgrs:grid_square_benchmark": ["DV", "DV"], - "properties.grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], - "properties.view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], - "properties.view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], - "properties.s2:degraded_msi_data_percentage_benchmark": [0, 0], - "properties.s2:nodata_pixel_percentage_benchmark": [0, 0], - "properties.s2:saturated_defective_pixel_percentage_benchmark": [0, 0], - "properties.s2:dark_features_percentage_benchmark": [0.619855, 0.411445], - "properties.s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], - "properties.s2:vegetation_percentage_benchmark": [13.550997, 13.711172], - "properties.s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], - "properties.s2:water_percentage_benchmark": [0.095783, 0.093679], - "properties.s2:unclassified_percentage_benchmark": [0.236433, 2.878225], - "properties.s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], - "properties.s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], - "properties.s2:thin_cirrus_percentage_benchmark": [0, 0], - "properties.s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], - "properties.s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], - "properties.s2:processing_baseline_benchmark": ["05.00", "02.14"], - "properties.s2:product_uri_benchmark": [ - "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", - "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", - ], - "properties.s2:generation_time_benchmark": [ - "2023-05-10T05:34:45.000000Z", - "2020-04-03T22:01:05.000000Z", - ], - "properties.s2:datatake_id_benchmark": [ - "GS2A_20200403T173901_024977_N05.00", - "GS2A_20200403T173901_024977_N02.14", - ], - "properties.s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], - "properties.s2:datastrip_id_benchmark": [ - "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", - "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", - ], - "properties.s2:granule_id_benchmark": [ - "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", - "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", - ], - "properties.s2:reflectance_conversion_factor_benchmark": [ - 1.00241535908783, - 1.00241535908783, - ], - "properties.datetime_benchmark": [ - "2020-04-03T17:54:07.524000Z", - "2020-04-03T17:54:07.524000Z", - ], - "properties.s2:sequence_benchmark": ["1", "0"], - "properties.earthsearch:s3_path_benchmark": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", - ], - "properties.earthsearch:payload_id_benchmark": [ - "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", - "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", - ], - "properties.earthsearch:boa_offset_applied_benchmark": [True, False], - "properties.processing:software.sentinel2-to-stac_benchmark": [ - "0.1.1", - "0.1.0", - ], - "properties.updated_benchmark": [ - "2023-10-08T00:32:51.304Z", - "2022-11-06T07:21:36.990Z", - ], - "geometry.type_benchmark": ["Polygon", "Polygon"], - "geometry.coordinates_benchmark": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.09828205302668, 35.1499211736146], - [-104.89285176524281, 35.154851672138626], - [-104.89152152018616, 36.14484027029347], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-104.89152152018616, 36.14484027029347], - [-104.89285176524281, 35.154851672138626], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "href_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "title_benchmark": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_benchmark": [[5490, 5490], [5490, 5490]], - "proj:transform_benchmark": [ - [20, 0, 399960, 0, -20, 4000020], - [20, 0, 399960, 0, -20, 4000020], - ], - "raster:bands_benchmark": [ - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - ], - "roles_benchmark": [["data", "reflectance"], ["data", "reflectance"]], - "asset_benchmark": ["aot", "aot"], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "band": ["1", "1"], - "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], - "mean_absolute_error": [25.393386840820312, 11.947012901306152], - "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], - }, - { - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "compare_id": [1, 2], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "band": ["1", "1"], - "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], - "mean_absolute_error": [25.393386840820312, 11.947012901306152], - "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], - }, - { - "type_candidate": ["Feature", "Feature"], - "stac_version_candidate": ["1.0.0", "1.0.0"], - "id_candidate": ["S2B_13SDV_20200401_1_L2A", "S2B_13SDV_20200401_0_L2A"], - "links_candidate": [ - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/S2B_13SDV_20200401_1_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_1_L2A/thumbnail", - }, - ], - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/S2B_13SDV_20200401_0_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2B_13SDV_20200401_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2B_13SDV_20200401_0_L2A/thumbnail", - }, - ], - ], - "bbox_candidate": [ - [ - -106.11191385205835, - 35.150822790058335, - -105.53827534157463, - 36.14367977962539, - ], - [ - -106.11191385205835, - 35.1499211736146, - -105.53527335203748, - 36.14369323431527, - ], - ], - "stac_extensions_candidate": [ - [ - "https://stac-extensions.github.io/projection/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/eo/v1.1.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - ], - [ - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/eo/v1.0.0/schema.json", - "https://stac-extensions.github.io/projection/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - ], - ], - "collection_candidate": ["sentinel-2-l2a", "sentinel-2-l2a"], - "properties.created_candidate": [ - "2023-10-07T12:09:07.273Z", - "2022-11-06T10:14:16.681Z", - ], - "properties.platform_candidate": ["sentinel-2b", "sentinel-2b"], - "properties.constellation_candidate": ["sentinel-2", "sentinel-2"], - "properties.instruments_candidate": [["msi"], ["msi"]], - "properties.eo:cloud_cover_candidate": [26.25798, 26.031335], - "properties.proj:epsg_candidate": [32613, 32613], - "properties.mgrs:latitude_band_candidate": ["S", "S"], - "properties.mgrs:grid_square_candidate": ["DV", "DV"], - "properties.grid:code_candidate": ["MGRS-13SDV", "MGRS-13SDV"], - "properties.view:sun_azimuth_candidate": [151.651828166802, 151.650282024803], - "properties.view:sun_elevation_candidate": [56.1648077338387, 56.164320019392], - "properties.s2:degraded_msi_data_percentage_candidate": [0.0165, 0.0], - "properties.s2:nodata_pixel_percentage_candidate": [66.295946, 66.037267], - "properties.s2:saturated_defective_pixel_percentage_candidate": [0, 0], - "properties.s2:dark_features_percentage_candidate": [2.557362, 4.251513], - "properties.s2:cloud_shadow_percentage_candidate": [0.709105, 1.063333], - "properties.s2:vegetation_percentage_candidate": [6.046846, 6.331863], - "properties.s2:not_vegetated_percentage_candidate": [55.535203, 53.858513], - "properties.s2:water_percentage_candidate": [0.019934, 0.045153], - "properties.s2:unclassified_percentage_candidate": [5.82255, 5.403911], - "properties.s2:medium_proba_clouds_percentage_candidate": [5.997636, 6.010647], - "properties.s2:high_proba_clouds_percentage_candidate": [20.260343, 20.020688], - "properties.s2:thin_cirrus_percentage_candidate": [0, 0], - "properties.s2:snow_ice_percentage_candidate": [3.051021, 3.01438], - "properties.s2:product_type_candidate": ["S2MSI2A", "S2MSI2A"], - "properties.s2:processing_baseline_candidate": ["05.00", "02.14"], - "properties.s2:product_uri_candidate": [ - "S2B_MSIL2A_20200401T174909_N0500_R141_T13SDV_20230624T053545.SAFE", - "S2B_MSIL2A_20200401T174909_N0214_R141_T13SDV_20200401T220155.SAFE", - ], - "properties.s2:generation_time_candidate": [ - "2023-06-24T05:35:45.000000Z", - "2020-04-01T22:01:55.000000Z", - ], - "properties.s2:datatake_id_candidate": [ - "GS2B_20200401T174909_016040_N05.00", - "GS2B_20200401T174909_016040_N02.14", - ], - "properties.s2:datatake_type_candidate": ["INS-NOBS", "INS-NOBS"], - "properties.s2:datastrip_id_candidate": [ - "S2B_OPER_MSI_L2A_DS_S2RP_20230624T053545_S20200401T175716_N05.00", - "S2B_OPER_MSI_L2A_DS_EPAE_20200401T220155_S20200401T175716_N02.14", - ], - "properties.s2:granule_id_candidate": [ - "S2B_OPER_MSI_L2A_TL_S2RP_20230624T053545_A016040_T13SDV_N05.00", - "S2B_OPER_MSI_L2A_TL_EPAE_20200401T220155_A016040_T13SDV_N02.14", - ], - "properties.s2:reflectance_conversion_factor_candidate": [ - 1.00356283682453, - 1.00356283682453, - ], - "properties.datetime_candidate": [ - "2020-04-01T18:04:04.327000Z", - "2020-04-01T18:04:04.327000Z", - ], - "properties.s2:sequence_candidate": ["1", "0"], - "properties.earthsearch:s3_path_candidate": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A", - ], - "properties.earthsearch:payload_id_candidate": [ - "roda-sentinel2/workflow-sentinel2-to-stac/71c0289236fa3e831ac2f8c860df8cae", - "roda-sentinel2/workflow-sentinel2-to-stac/afb43c585d466972865ed5139ba35520", - ], - "properties.earthsearch:boa_offset_applied_candidate": [True, False], - "properties.processing:software.sentinel2-to-stac_candidate": [ - "0.1.1", - "0.1.0", - ], - "properties.updated_candidate": [ - "2023-10-07T12:09:07.273Z", - "2022-11-06T10:14:16.681Z", - ], - "geometry.type_candidate": ["Polygon", "Polygon"], - "geometry.coordinates_candidate": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.0982941692468, 35.150822790058335], - [-105.85393882465051, 35.15385918076215], - [-105.68102037327243, 35.69893282033011], - [-105.59450557216445, 35.97641506053815], - [-105.56226433775963, 36.07584727804726], - [-105.53827534157463, 36.14367977962539], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-105.53527335203748, 36.14369323431527], - [-105.56579262097148, 36.05653228802631], - [-105.68980719734964, 35.659112338538634], - [-105.85157080324588, 35.15190642354915], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "href_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "title_candidate": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_candidate": [[5490, 5490], [5490, 5490]], - "proj:transform_candidate": [ - [20, 0, 399960, 0, -20, 4000020], - [20, 0, 399960, 0, -20, 4000020], - ], - "raster:bands_candidate": [ - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - ], - "roles_candidate": [["data", "reflectance"], ["data", "reflectance"]], - "asset_candidate": ["aot", "aot"], - "compare_id": [1, 2], - "map_id_candidate": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2B_13SDV_20200401_0_L2A/AOT.tif", - ], - "type_benchmark": ["Feature", "Feature"], - "stac_version_benchmark": ["1.0.0", "1.0.0"], - "id_benchmark": ["S2A_13SDV_20200403_1_L2A", "S2A_13SDV_20200403_0_L2A"], - "links_benchmark": [ - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/S2A_13SDV_20200403_1_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_1_L2A/thumbnail", - }, - ], - [ - { - "rel": "self", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A", - "type": "application/geo+json", - }, - { - "rel": "canonical", - "href": "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/S2A_13SDV_20200403_0_L2A.json", - "type": "application/json", - }, - { - "rel": "license", - "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", - }, - { - "rel": "derived_from", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l1c/items/S2A_13SDV_20200403_0_L1C", - "type": "application/geo+json", - }, - { - "rel": "parent", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "collection", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a", - "type": "application/json", - }, - { - "rel": "root", - "href": "https://earth-search.aws.element84.com/v1", - "type": "application/json", - "title": "Earth Search by Element 84", - }, - { - "rel": "thumbnail", - "href": "https://earth-search.aws.element84.com/v1/collections/sentinel-2-l2a/items/S2A_13SDV_20200403_0_L2A/thumbnail", - }, - ], - ], - "bbox_benchmark": [ - [ - -106.11191385205835, - 35.1499211736146, - -104.89152152018616, - 36.14484027029347, - ], - [ - -106.11191385205835, - 35.1499211736146, - -104.89152152018616, - 36.14484027029347, - ], - ], - "stac_extensions_benchmark": [ - [ - "https://stac-extensions.github.io/projection/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/eo/v1.1.0/schema.json", - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - ], - [ - "https://stac-extensions.github.io/grid/v1.0.0/schema.json", - "https://stac-extensions.github.io/view/v1.0.0/schema.json", - "https://stac-extensions.github.io/eo/v1.0.0/schema.json", - "https://stac-extensions.github.io/projection/v1.0.0/schema.json", - "https://stac-extensions.github.io/raster/v1.1.0/schema.json", - "https://stac-extensions.github.io/processing/v1.1.0/schema.json", - "https://stac-extensions.github.io/mgrs/v1.0.0/schema.json", - ], - ], - "collection_benchmark": ["sentinel-2-l2a", "sentinel-2-l2a"], - "properties.created_benchmark": [ - "2023-10-08T00:32:51.304Z", - "2022-11-06T07:21:36.990Z", - ], - "properties.platform_benchmark": ["sentinel-2a", "sentinel-2a"], - "properties.constellation_benchmark": ["sentinel-2", "sentinel-2"], - "properties.instruments_benchmark": [["msi"], ["msi"]], - "properties.eo:cloud_cover_benchmark": [0.394644, 0.946059], - "properties.proj:epsg_benchmark": [32613, 32613], - "properties.mgrs:latitude_band_benchmark": ["S", "S"], - "properties.mgrs:grid_square_benchmark": ["DV", "DV"], - "properties.grid:code_benchmark": ["MGRS-13SDV", "MGRS-13SDV"], - "properties.view:sun_azimuth_benchmark": [147.277474939279, 147.275985208519], - "properties.view:sun_elevation_benchmark": [55.8900570304106, 55.8895151168133], - "properties.s2:degraded_msi_data_percentage_benchmark": [0, 0], - "properties.s2:nodata_pixel_percentage_benchmark": [0, 0], - "properties.s2:saturated_defective_pixel_percentage_benchmark": [0, 0], - "properties.s2:dark_features_percentage_benchmark": [0.619855, 0.411445], - "properties.s2:cloud_shadow_percentage_benchmark": [0.074044, 0.143752], - "properties.s2:vegetation_percentage_benchmark": [13.550997, 13.711172], - "properties.s2:not_vegetated_percentage_benchmark": [79.290646, 77.119505], - "properties.s2:water_percentage_benchmark": [0.095783, 0.093679], - "properties.s2:unclassified_percentage_benchmark": [0.236433, 2.878225], - "properties.s2:medium_proba_clouds_percentage_benchmark": [0.262985, 0.760678], - "properties.s2:high_proba_clouds_percentage_benchmark": [0.131658, 0.185381], - "properties.s2:thin_cirrus_percentage_benchmark": [0, 0], - "properties.s2:snow_ice_percentage_benchmark": [5.737595, 4.696162], - "properties.s2:product_type_benchmark": ["S2MSI2A", "S2MSI2A"], - "properties.s2:processing_baseline_benchmark": ["05.00", "02.14"], - "properties.s2:product_uri_benchmark": [ - "S2A_MSIL2A_20200403T173901_N0500_R098_T13SDV_20230510T053445.SAFE", - "S2A_MSIL2A_20200403T173901_N0214_R098_T13SDV_20200403T220105.SAFE", - ], - "properties.s2:generation_time_benchmark": [ - "2023-05-10T05:34:45.000000Z", - "2020-04-03T22:01:05.000000Z", - ], - "properties.s2:datatake_id_benchmark": [ - "GS2A_20200403T173901_024977_N05.00", - "GS2A_20200403T173901_024977_N02.14", - ], - "properties.s2:datatake_type_benchmark": ["INS-NOBS", "INS-NOBS"], - "properties.s2:datastrip_id_benchmark": [ - "S2A_OPER_MSI_L2A_DS_S2RP_20230510T053445_S20200403T174815_N05.00", - "S2A_OPER_MSI_L2A_DS_SGS__20200403T220105_S20200403T174815_N02.14", - ], - "properties.s2:granule_id_benchmark": [ - "S2A_OPER_MSI_L2A_TL_S2RP_20230510T053445_A024977_T13SDV_N05.00", - "S2A_OPER_MSI_L2A_TL_SGS__20200403T220105_A024977_T13SDV_N02.14", - ], - "properties.s2:reflectance_conversion_factor_benchmark": [ - 1.00241535908783, - 1.00241535908783, - ], - "properties.datetime_benchmark": [ - "2020-04-03T17:54:07.524000Z", - "2020-04-03T17:54:07.524000Z", - ], - "properties.s2:sequence_benchmark": ["1", "0"], - "properties.earthsearch:s3_path_benchmark": [ - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A", - "s3://sentinel-cogs/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A", - ], - "properties.earthsearch:payload_id_benchmark": [ - "roda-sentinel2/workflow-sentinel2-to-stac/5c5486e239b6fb66c09401d57834e542", - "roda-sentinel2/workflow-sentinel2-to-stac/00bfe5eff22b9aae0b5adfae696a11fa", - ], - "properties.earthsearch:boa_offset_applied_benchmark": [True, False], - "properties.processing:software.sentinel2-to-stac_benchmark": [ - "0.1.1", - "0.1.0", - ], - "properties.updated_benchmark": [ - "2023-10-08T00:32:51.304Z", - "2022-11-06T07:21:36.990Z", - ], - "geometry.type_benchmark": ["Polygon", "Polygon"], - "geometry.coordinates_benchmark": [ - [ - [ - [-106.11191385205835, 36.13972769324406], - [-106.09828205302668, 35.1499211736146], - [-104.89285176524281, 35.154851672138626], - [-104.89152152018616, 36.14484027029347], - [-106.11191385205835, 36.13972769324406], - ] - ], - [ - [ - [-106.11191385205835, 36.13972769324406], - [-104.89152152018616, 36.14484027029347], - [-104.89285176524281, 35.154851672138626], - [-106.09828205302668, 35.1499211736146], - [-106.11191385205835, 36.13972769324406], - ] - ], - ], - "href_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "title_benchmark": [ - "Aerosol optical thickness (AOT)", - "Aerosol optical thickness (AOT)", - ], - "proj:shape_benchmark": [[5490, 5490], [5490, 5490]], - "proj:transform_benchmark": [ - [20, 0, 399960, 0, -20, 4000020], - [20, 0, 399960, 0, -20, 4000020], - ], - "raster:bands_benchmark": [ - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - [ - { - "nodata": 0, - "data_type": "uint16", - "bits_per_sample": 15, - "spatial_resolution": 20, - "scale": 0.001, - "offset": 0, - } - ], - ], - "roles_benchmark": [["data", "reflectance"], ["data", "reflectance"]], - "asset_benchmark": ["aot", "aot"], - "map_id_benchmark": [ - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_1_L2A/AOT.tif", - "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/13/S/DV/2020/4/S2A_13SDV_20200403_0_L2A/AOT.tif", - ], - "band": ["1", "1"], - "coefficient_of_determination": [-1.449866533279419, -0.46196842193603516], - "mean_absolute_error": [25.393386840820312, 11.947012901306152], - "mean_absolute_percentage_error": [0.2044084370136261, 0.15074075758457184], - }, + f"{TEST_DATA_DIR}/expected_catalog_df_no_filter.pkl", + f"{TEST_DATA_DIR}/expected_catalog_df_allow_list.pkl", + f"{TEST_DATA_DIR}/expected_catalog_df_block_list.pkl", ] allow_list = [None, ["map_id", "compare_id"], None] @@ -1583,25 +505,25 @@ def case_stac_catalog_comparison_success( assets, allow_list, block_list, - pd.DataFrame(expected_catalog_df), + pd.read_pickle(expected_catalog_df), ) bad_times = ["2020-04-01"] -bad_assets = [["surface_water"], None, None] -exceptions = [ValueError, KeyError, KeyError] -bad_allow_list = [None, ["arb"], None] -bad_block_list = [None, None, ["arb"]] +bad_assets = [["surface_water"], None, None, None] +exceptions = [ValueError, ValueError, KeyError, KeyError] +bad_allow_list = [None, ["arb"], ["arb"], None] +bad_block_list = [None, ["arb"], None, ["arb"]] @parametrize( "url, collection, time, bbox, assets, allow_list, block_list, exception", list( zip( - [url] * 3, - [collection] * 3, - bad_times * 3, - [bbox] * 3, + [url] * len(bad_assets), + [collection] * len(bad_assets), + bad_times * len(bad_assets), + [bbox] * len(bad_assets), bad_assets, bad_allow_list, bad_block_list, From 03b27c345f4f7ddc2f0bdc7e916897a3166ffd14 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 18:07:58 -0400 Subject: [PATCH 08/12] Add s3fs --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b0d65de9..bf33f2e3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 +s3fs==2024.3.1 From b034e18cc9a137e0a8fe0c79159e1dfff282c486 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 18:23:24 -0400 Subject: [PATCH 09/12] Adjust s3fs version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index bf33f2e3..3c4ee98d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 -s3fs==2024.3.1 +s3fs==2023.1.1 From 2d4345e3c5c14ae0ac0264ffd53c290c64db7626 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 18:30:56 -0400 Subject: [PATCH 10/12] s3fs version correction --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3c4ee98d..cd83dc94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 -s3fs==2023.1.1 +s3fs==2023.5.0 From 913b9587b28e8bbad39cd6fc3194e29035e53ccb Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 18:50:53 -0400 Subject: [PATCH 11/12] s3fs version wild card range --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cd83dc94..0ca6c33b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 -s3fs==2023.5.0 +s3fs From 1e9c1db92ba3f3375cbc8aedeae004e8e741db72 Mon Sep 17 00:00:00 2001 From: Gregory Petrochenkov Date: Mon, 25 Mar 2024 19:05:51 -0400 Subject: [PATCH 12/12] s3fs less or equal version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ca6c33b..ac3ed7ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ flox==0.7.2 xskillscore==0.0.24 pyogrio==0.7.2 pystac-client==0.7.5 -s3fs +s3fs<=2023.12.1