diff --git a/catalogs/trilateral.yaml b/catalogs/trilateral.yaml index 0d060444..5a941407 100644 --- a/catalogs/trilateral.yaml +++ b/catalogs/trilateral.yaml @@ -34,6 +34,9 @@ collections: - SIC_sea_ice_thickness_cryosat - NASAPopulation - WSF_world_settlement_footprint + - N12_sea_ice_concentration_antarctic + - N12_1_sea_ice_concentration_arctic + - N12_2_sea_ice_concentration_both_hemispheres - E10e_ndvi - N11_ocean_primary_productivity - SMC_soil_moisture_anomaly diff --git a/collections/N12_1_sea_ice_concentration_arctic.yaml b/collections/N12_1_sea_ice_concentration_arctic.yaml new file mode 100644 index 00000000..a1d90b6b --- /dev/null +++ b/collections/N12_1_sea_ice_concentration_arctic.yaml @@ -0,0 +1,37 @@ +Name: N12_1_sea_ice_concentration_arctic +Title: Sea ice concentration GCOM-W - Arctic +EodashIdentifier: N12_1_sea_ice_concentration_arctic +Description: N12_sea_ice_concentration/N12.md +Themes: + - cryosphere +Tags: + - sea ice +Satellite: + - GCOM-W +Sensor: + - GCOM-W +Agency: + - JAXA +Provider: + - Name: "Global Change Observation Mission – Water SHIZUKU (GCOM-W)" + Url: https://global.jaxa.jp/projects/sat/gcom_w/ + Description: The purpose of this website is, first and foremost, to provide the science and broader communities with global, updated ocean productivity estimates. In addition, the website provides information on the models employed, access to model code and ancillary data sets, and comparisons of productivity estimates for alternative models. Field data sets relevant to ocean productivity modeling are also provided and will be continually expanded as new data become available. Educational material and background information on phytoplankton ecology and physiology will also be added in the future. + Roles: + - producer + - Name: GCOM-W Research Product Distribution Service + Url: https://suzaku.eorc.jaxa.jp/GCOM_W/research/resdist.html + Description: The GCOM-W Research Product Distribution Service provides GCOM-W/AMSR2 and Aqua/AMSR-E research products with free of charge. + Roles: + - host +Resources: + - EndPoint: https://ogcpreview2.restecmap.com/examind/api/WS/wms/default + Name: WMS + Type: OverwriteTimes + LayerId: SIC_N + OverwriteBBox: [-20, 77, 50, 83] + Disable_Items: true + DateTimeInterval: + Start: "1978-11-01T00:00:00" + End: "2023-12-31T00:00:00" +Legend: N12_sea_ice_concentration/World-SIC.png +Image: N12_sea_ice_concentration/Arctic-N12_1.png diff --git a/collections/N12_2_sea_ice_concentration_both_hemispheres.yaml b/collections/N12_2_sea_ice_concentration_both_hemispheres.yaml new file mode 100644 index 00000000..3edcacce --- /dev/null +++ b/collections/N12_2_sea_ice_concentration_both_hemispheres.yaml @@ -0,0 +1,36 @@ +Name: N12_2_sea_ice_concentration_both_hemispheres +Title: Sea ice concentration GCOM-W - Global +EodashIdentifier: N12_2_sea_ice_concentration_both_hemispheres +Description: N12_sea_ice_concentration/N12.md +Themes: + - cryosphere +Tags: + - sea ice +Satellite: + - GCOM-W +Sensor: + - GCOM-W +Agency: + - JAXA +Provider: + - Name: "Global Change Observation Mission – Water SHIZUKU (GCOM-W)" + Url: https://global.jaxa.jp/projects/sat/gcom_w/ + Description: The purpose of this website is, first and foremost, to provide the science and broader communities with global, updated ocean productivity estimates. In addition, the website provides information on the models employed, access to model code and ancillary data sets, and comparisons of productivity estimates for alternative models. Field data sets relevant to ocean productivity modeling are also provided and will be continually expanded as new data become available. Educational material and background information on phytoplankton ecology and physiology will also be added in the future. + Roles: + - producer + - Name: GCOM-W Research Product Distribution Service + Url: https://suzaku.eorc.jaxa.jp/GCOM_W/research/resdist.html + Description: The GCOM-W Research Product Distribution Service provides GCOM-W/AMSR2 and Aqua/AMSR-E research products with free of charge. + Roles: + - host +Resources: + - EndPoint: https://ogcpreview2.restecmap.com/examind/api/WS/wms/default + Name: WMS + Type: OverwriteTimes + LayerId: SIC_N + Disable_Items: true + DateTimeInterval: + Start: "1978-11-01T00:00:00" + End: "2023-12-31T00:00:00" +Legend: N12_sea_ice_concentration/World-SIC.png +Image: N12_sea_ice_concentration/Arctic-N12_1.png diff --git a/collections/N12_sea_ice_concentration_antarctic.yaml b/collections/N12_sea_ice_concentration_antarctic.yaml new file mode 100644 index 00000000..5aae99cd --- /dev/null +++ b/collections/N12_sea_ice_concentration_antarctic.yaml @@ -0,0 +1,37 @@ +Name: N12_sea_ice_concentration_antarctic +Title: Sea ice concentration GCOM-W - Antarctic +EodashIdentifier: N12_sea_ice_concentration_antarctic +Description: N12_sea_ice_concentration/N12.md +Themes: + - cryosphere +Tags: + - sea ice +Satellite: + - GCOM-W +Sensor: + - GCOM-W +Agency: + - JAXA +Provider: + - Name: "Global Change Observation Mission – Water SHIZUKU (GCOM-W)" + Url: https://global.jaxa.jp/projects/sat/gcom_w/ + Description: The purpose of this website is, first and foremost, to provide the science and broader communities with global, updated ocean productivity estimates. In addition, the website provides information on the models employed, access to model code and ancillary data sets, and comparisons of productivity estimates for alternative models. Field data sets relevant to ocean productivity modeling are also provided and will be continually expanded as new data become available. Educational material and background information on phytoplankton ecology and physiology will also be added in the future. + Roles: + - producer + - Name: GCOM-W Research Product Distribution Service + Url: https://suzaku.eorc.jaxa.jp/GCOM_W/research/resdist.html + Description: The GCOM-W Research Product Distribution Service provides GCOM-W/AMSR2 and Aqua/AMSR-E research products with free of charge. + Roles: + - host +Resources: + - EndPoint: https://ogcpreview2.restecmap.com/examind/api/WS/wms/default + Name: WMS + Type: OverwriteTimes + LayerId: SIC_S + OverwriteBBox: [-107, -84, 125, -64] + Disable_Items: true + DateTimeInterval: + Start: "1978-11-01T00:00:00" + End: "2023-12-31T00:00:00" +Legend: N12_sea_ice_concentration/World-SIC.png +Image: N12_sea_ice_concentration/Antarctic-N12.png diff --git a/generators/generate_indicators.py b/generators/generate_indicators.py index b74bd263..54371bbd 100644 --- a/generators/generate_indicators.py +++ b/generators/generate_indicators.py @@ -85,7 +85,7 @@ def process_catalog_file(file_path, options): # For full catalog save with items this still seems to be faster catalog.save(dest_href="../build/%s"%config["id"]) end = time.time() - print("Time consumed in saving: ", end - start) + print(f"Catalog {config['id']}: Time consumed in saving: {end - start}") if options.vd: # try to validate catalog if flag was set @@ -121,90 +121,37 @@ def process_collection_file(config, file_path, catalog): def handle_collection_only(config, endpoint, data, catalog): - times = [] - collection = get_or_create_collection(catalog, data["Name"], data, config, endpoint) - if endpoint.get("Type") == "OverwriteTimes" and endpoint.get("Times"): - times = endpoint["Times"] - elif endpoint.get("Type") == "OverwriteTimes" and endpoint.get("DateTimeInterval"): - start = endpoint["DateTimeInterval"].get("Start", "2020-09-01T00:00:00") - end = endpoint["DateTimeInterval"].get("End", "2020-10-01T00:00:00") - timedelta_config = endpoint["DateTimeInterval"].get("Timedelta", {'days': 1}) - times = generateDateIsostringsFromInterval(start, end, timedelta_config) - for t in times: - item = Item( - id = t, - bbox=endpoint.get("OverwriteBBox"), - properties={}, - geometry = None, - datetime = parser.isoparse(t), - ) - link = collection.add_item(item) - link.extra_fields["datetime"] = t - if len(times) > 0: - collection.update_extent_from_items() + collection, times = get_or_create_collection(catalog, data["Name"], data, config, endpoint) + if len(times) > 0 and not endpoint.get("Disable_Items"): + for t in times: + item = Item( + id = t, + bbox=endpoint.get("OverwriteBBox"), + properties={}, + geometry = None, + datetime = parser.isoparse(t), + ) + link = collection.add_item(item) + link.extra_fields["datetime"] = t add_collection_information(config, collection, data) add_to_catalog(collection, catalog, None, data) - -def handle_GeoDB_Tiles_endpoint(config, endpoint, data, catalog): - if "Dates" in data: - pass - else: - select = "?select=%s"%endpoint["TimeKey"] - url = endpoint["DBEndpoint"] + endpoint["Database"] + "_%s"%endpoint["Source"] + select - response = json.loads(requests.get(url).text) - times = set([entry[endpoint["TimeKey"]] for entry in response]) - if len(times) > 0: - # Create an item per time to allow visualization in stac clients - styles = None - if hasattr(endpoint, "Styles"): - styles = endpoint["Styles"] - collection = get_or_create_collection(catalog, data["Name"], data, config, endpoint) - # TODO: For now we create global extent, we should be able to - # fetch the extent of the layer - for t in times: - item = Item( - id = t, - bbox=[-180.0, -90.0, 180.0, 90.0], - properties={}, - geometry = None, - datetime = parser.isoparse(t), - ) - # add_visualization_info(item, data, endpoint, time=t, styles=styles) - link = collection.add_item(item) - link.extra_fields["datetime"] = t - collection.update_extent_from_items() - add_visualization_info(collection, data, endpoint, styles=styles) - add_collection_information(config, collection, data) - add_to_catalog(collection, catalog, endpoint, data) - - def handle_WMS_endpoint(config, endpoint, data, catalog): - times = [] - extent = retrieveExtentFromWMS(endpoint["EndPoint"], endpoint["LayerId"]) - if endpoint.get("Type") == "OverwriteTimes" and endpoint.get("Times"): - times = endpoint["Times"] - elif endpoint.get("Type") == "OverwriteTimes" and endpoint.get("DateTimeInterval"): - start = endpoint["DateTimeInterval"].get("Start", "2020-09-01T00:00:00") - end = endpoint["DateTimeInterval"].get("End", "2020-10-01T00:00:00") - timedelta_config = endpoint["DateTimeInterval"].get("Timedelta", {'days': 1}) - times = generateDateIsostringsFromInterval(start, end, timedelta_config) - else: - times = extent["temporal"] - if "OverwriteBBox" in endpoint: - extent["spatial"] = endpoint["OverwriteBBox"] - - collection = get_or_create_collection(catalog, data["Name"], data, config, endpoint) + collection, times = get_or_create_collection(catalog, data["Name"], data, config, endpoint) + spatial_extent = collection.extent.spatial.to_dict().get("bbox", [-180, -90, 180, 90])[0] + if not endpoint.get("Type") == "OverwriteTimes" and not endpoint.get("OverwriteBBox"): + spatial_extent, times = retrieveExtentFromWMS(endpoint["EndPoint"], endpoint["LayerId"]) + # Create an item per time to allow visualization in stac clients styles = None if hasattr(endpoint, "Styles"): styles = endpoint["Styles"] - if len(times) > 0: + if len(times) > 0 and not endpoint.get("Disable_Items"): for t in times: item = Item( id = t, - bbox=extent["spatial"], + bbox=spatial_extent, properties={}, geometry = None, datetime = parser.isoparse(t), @@ -212,8 +159,8 @@ def handle_WMS_endpoint(config, endpoint, data, catalog): add_visualization_info(item, data, endpoint, time=t, styles=styles) link = collection.add_item(item) link.extra_fields["datetime"] = t + collection.update_extent_from_items() - collection.update_extent_from_items() add_visualization_info(collection, data, endpoint, styles=styles) add_collection_information(config, collection, data) add_to_catalog(collection, catalog, endpoint, data) @@ -245,7 +192,7 @@ def get_or_create_collection(catalog, collection_id, data, config, endpoint): # Check if collection already in catalog for collection in catalog.get_collections(): if collection.id == collection_id: - return collection + return collection, [] # If none found create a new one spatial_extent = [-180.0, -90.0, 180.0, 90.0] if endpoint.get("OverwriteBBox"): @@ -253,7 +200,20 @@ def get_or_create_collection(catalog, collection_id, data, config, endpoint): spatial_extent = SpatialExtent([ spatial_extent, ]) + times = [] temporal_extent = TemporalExtent([[datetime.now(), None]]) + if endpoint.get("Type") == "OverwriteTimes": + if endpoint.get("Times"): + times = endpoint.get("Times") + times_datetimes = sorted([parser.isoparse(time) for time in times]) + temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]]) + elif endpoint.get("DateTimeInterval"): + start = endpoint["DateTimeInterval"].get("Start", "2020-09-01T00:00:00") + end = endpoint["DateTimeInterval"].get("End", "2020-10-01T00:00:00") + timedelta_config = endpoint["DateTimeInterval"].get("Timedelta", {'days': 1}) + times = generateDateIsostringsFromInterval(start, end, timedelta_config) + times_datetimes = sorted([parser.isoparse(time) for time in times]) + temporal_extent = TemporalExtent([[times_datetimes[0], times_datetimes[-1]]]) extent = Extent(spatial=spatial_extent, temporal=temporal_extent) # Check if description is link to markdown file @@ -294,7 +254,7 @@ def get_or_create_collection(catalog, collection_id, data, config, endpoint): ], extent=extent ) - return collection + return (collection, times) def add_to_catalog(collection, catalog, endpoint, data): # check if already in catalog, if it is do not re-add it @@ -335,7 +295,7 @@ def add_to_catalog(collection, catalog, endpoint, data): def handle_GeoDB_endpoint(config, endpoint, data, catalog): - collection = get_or_create_collection(catalog, endpoint["CollectionId"], data, config, endpoint) + collection, _ = get_or_create_collection(catalog, endpoint["CollectionId"], data, config, endpoint) select = "?select=aoi,aoi_id,country,city,time" url = endpoint["EndPoint"] + endpoint["Database"] + "_%s"%endpoint["CollectionId"] + select if additional_query_parameters := endpoint.get("AdditionalQueryString"): @@ -408,7 +368,7 @@ def handle_GeoDB_endpoint(config, endpoint, data, catalog): def handle_STAC_based_endpoint(config, endpoint, data, catalog, headers=None): if "Locations" in data: - root_collection = get_or_create_collection(catalog, data["Name"], data, config, endpoint) + root_collection, _ = get_or_create_collection(catalog, data["Name"], data, config, endpoint) for location in data["Locations"]: collection = process_STACAPI_Endpoint( config=config, @@ -658,7 +618,7 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None print("Visualization endpoint not supported") def process_STACAPI_Endpoint(config, endpoint, data, catalog, headers={}, bbox=None, root_collection=None): - collection = get_or_create_collection(catalog, endpoint["CollectionId"], data, config, endpoint) + collection, _ = get_or_create_collection(catalog, endpoint["CollectionId"], data, config, endpoint) add_visualization_info(collection, data, endpoint) api = Client.open(endpoint["EndPoint"], headers=headers) @@ -751,7 +711,7 @@ def generate_thumbnail(stac_object, data, endpoint, file_url=None, time=None, st def process_STAC_Datacube_Endpoint(config, endpoint, data, catalog): - collection = get_or_create_collection(catalog, data["Name"], data, config, endpoint) + collection, _ = get_or_create_collection(catalog, data["Name"], data, config, endpoint) add_visualization_info(collection, data, endpoint) stac_endpoint_url = endpoint["EndPoint"] @@ -819,7 +779,8 @@ def add_collection_information(config, collection, data): print("WARNING: License could not be parsed, falling back to proprietary") collection.license = "proprietary" else: - print("WARNING: No license was provided, falling back to proprietary") + # print("WARNING: No license was provided, falling back to proprietary") + pass if "Provider" in data: try: diff --git a/generators/utils.py b/generators/utils.py index cf17ff7c..b92877ac 100644 --- a/generators/utils.py +++ b/generators/utils.py @@ -71,11 +71,7 @@ def retrieveExtentFromWMS(capabilties_url, layer): bbox = [-180,-90,180,90] if wms and wms[layer].boundingBoxWGS84: bbox = [float(x) for x in wms[layer].boundingBoxWGS84] - - return { - "spatial": bbox, - "temporal": times, - } + return bbox, times def interval(start: datetime, stop: datetime, delta: timedelta) -> Iterator[datetime]: while start <= stop: