From d1057acedbb462ceda6ebb6ece4f1c920a4e8eea Mon Sep 17 00:00:00 2001 From: Lubomir Dolezal Date: Tue, 19 Dec 2023 11:49:16 +0100 Subject: [PATCH 1/2] split mobility data for RACE and trilateral --- catalogs/race.yaml | 2 +- catalogs/trilateral.yaml | 1 + .../E12c_truck_detections_motorways_GTIF.yaml | 2 +- ...d_truck_detections_primary_roads_GTIF.yaml | 2 +- collections/GG_Google_Mobility_Data_RACE.yaml | 22 ++++++++++++++++++ ...> GG_Google_Mobility_Data_trilateral.yaml} | 3 ++- generators/generate_indicators.py | 23 ++++++++++--------- 7 files changed, 40 insertions(+), 15 deletions(-) create mode 100644 collections/GG_Google_Mobility_Data_RACE.yaml rename collections/{GG_Mobility_Data.yaml => GG_Google_Mobility_Data_trilateral.yaml} (86%) diff --git a/catalogs/race.yaml b/catalogs/race.yaml index 40b44736..de36bfc0 100644 --- a/catalogs/race.yaml +++ b/catalogs/race.yaml @@ -77,5 +77,5 @@ collections: - OX_Crude_Oil_Storage_Index - OX_EU_Crude_Oil_Storage_Index # - OW_Covid_19_vaccinations - - GG_Mobility_Data + - GG_Google_Mobility_Data_RACE # - CV_Covid_19_cases diff --git a/catalogs/trilateral.yaml b/catalogs/trilateral.yaml index 955c4a04..e731912c 100644 --- a/catalogs/trilateral.yaml +++ b/catalogs/trilateral.yaml @@ -86,3 +86,4 @@ collections: - ESDC_gross_primary_productivity - ESDC_kndvi - ESDC_net_ecosystem_exchange + - GG_Google_Mobility_Data_trilateral diff --git a/collections/E12c_truck_detections_motorways_GTIF.yaml b/collections/E12c_truck_detections_motorways_GTIF.yaml index 9f67677c..4c8f11ff 100644 --- a/collections/E12c_truck_detections_motorways_GTIF.yaml +++ b/collections/E12c_truck_detections_motorways_GTIF.yaml @@ -23,6 +23,6 @@ Resources: Type: byoc CollectionId: 34fbfb05-c3e5-4885-9840-830ee85dddd4 LayerId: VIS_TRUCK_DETECTION_MOTORWAYS_NEW - OverwriteBBox: [46.4, 49, 9.5, 17] + OverwriteBBox: [5.5, 44.0, 16.5, 48.7] Image: E12c_truck_detections_motorways_GTIF/thumbnail.png Legend: E12c_truck_detections_motorways_GTIF/cm_legend.png diff --git a/collections/E12d_truck_detections_primary_roads_GTIF.yaml b/collections/E12d_truck_detections_primary_roads_GTIF.yaml index 9aea533f..44bd41ab 100644 --- a/collections/E12d_truck_detections_primary_roads_GTIF.yaml +++ b/collections/E12d_truck_detections_primary_roads_GTIF.yaml @@ -23,6 +23,6 @@ Resources: Type: byoc CollectionId: 81201dac-c408-4cf8-b9a5-ca8349e4ab6c LayerId: VIS_TRUCK_DETECTION_PRIMARY_NEW - OverwriteBBox: [46.4, 49, 9.5, 17] + OverwriteBBox: [5.5, 44.0, 16.5, 48.7] Image: E12d_truck_detections_primary_roads_GTIF/thumbnail.png Legend: E12d_truck_detections_motorways_GTIF/cm_legend.png diff --git a/collections/GG_Google_Mobility_Data_RACE.yaml b/collections/GG_Google_Mobility_Data_RACE.yaml new file mode 100644 index 00000000..336fa0e8 --- /dev/null +++ b/collections/GG_Google_Mobility_Data_RACE.yaml @@ -0,0 +1,22 @@ +Name: GG_Google_Mobility_Data_RACE +Title: Mobility +EodashIdentifier: GG +Subtitle: Mobility data from Google +Description: GG_Google_Mobility_Data/GG.md +Themes: + - economy +Tags: + - placeholder +Satellite: + - placeholder +Sensor: + - placeholder +Agency: + - ESA +Resources: + - EndPoint: https://xcube-geodb.brockmann-consult.de/eodash/6bf15325-f6a0-4b6a-bf80-a2491753f8f2/ + Name: GeoDB + Database: eodash + CollectionId: Mobility_data + AdditionalQueryString: 'aoi_id=in.("AT","BA","BE","BG","CH","CZ","DE","DK","EE","EG","ES","FI","FR","GB","GR","HR","HU","IE","IT","LT","LU","LV","MK","MT","NL","NO","PL","PT","RO","RS","RU","SE","SI","SK","TR")' + IdKey: 'country' diff --git a/collections/GG_Mobility_Data.yaml b/collections/GG_Google_Mobility_Data_trilateral.yaml similarity index 86% rename from collections/GG_Mobility_Data.yaml rename to collections/GG_Google_Mobility_Data_trilateral.yaml index aa57ec51..b7318f8f 100644 --- a/collections/GG_Mobility_Data.yaml +++ b/collections/GG_Google_Mobility_Data_trilateral.yaml @@ -1,4 +1,4 @@ -Name: GG_Google_Mobility_Data +Name: GG_Google_Mobility_Data_trilateral Title: Mobility EodashIdentifier: GG Subtitle: Mobility data from Google @@ -18,3 +18,4 @@ Resources: Name: GeoDB Database: eodash CollectionId: Mobility_data + IdKey: 'country' diff --git a/generators/generate_indicators.py b/generators/generate_indicators.py index ccd34af5..f38e941f 100644 --- a/generators/generate_indicators.py +++ b/generators/generate_indicators.py @@ -337,10 +337,9 @@ def add_to_catalog(collection, catalog, endpoint, data): def handle_GeoDB_endpoint(config, endpoint, data, catalog): collection = get_or_create_collection(catalog, endpoint["CollectionId"], data, config, endpoint) select = "?select=aoi,aoi_id,country,city,time" - where_parameter = endpoint.get("WhereParameter") url = endpoint["EndPoint"] + endpoint["Database"] + "_%s"%endpoint["CollectionId"] + select - if where_parameter: - url += f"&{where_parameter}" + if additional_query_parameters := endpoint.get("AdditionalQueryString"): + url += f"&{additional_query_parameters}" response = json.loads(requests.get(url).text) # Sort locations by key @@ -354,15 +353,17 @@ def handle_GeoDB_endpoint(config, endpoint, data, catalog): unique_values = list({v["aoi_id"]:v for v in values}.values())[0] country = unique_values["country"] city = unique_values["city"] + IdKey = endpoint.get("IdKey", "city") + IdValue = unique_values[IdKey] if country not in countries: countries.append(country) - # sanitize city identifier to be sure it is filename save - if city is not None: - city = "".join([c for c in city if c.isalpha() or c.isdigit() or c==' ']).rstrip() - # Additional check to see if city name is empty afterwards - if city == "" or city is None: - # use aoi_id as a fallback unique id instead of city - city = key + # sanitize unique key identifier to be sure it is saveable as a filename + if IdValue is not None: + IdValue = "".join([c for c in IdValue if c.isalpha() or c.isdigit() or c==' ']).rstrip() + # Additional check to see if unique key name is empty afterwards + if IdValue == "" or IdValue is None: + # use aoi_id as a fallback unique id instead of configured key + IdValue = key if city not in cities: cities.append(city) min_date = min(times) @@ -373,7 +374,7 @@ def handle_GeoDB_endpoint(config, endpoint, data, catalog): buff = 0.01 bbox = [lon-buff, lat-buff,lon+buff,lat+buff] item = Item( - id = city, + id = IdValue, bbox=bbox, properties={}, geometry = create_geojson_point(lon, lat), From 9c9cd480267494f7f0ac6e4a936f98b2d0898640 Mon Sep 17 00:00:00 2001 From: Lubomir Dolezal Date: Tue, 19 Dec 2023 12:21:39 +0100 Subject: [PATCH 2/2] minor fix --- collections/GG_Google_Mobility_Data_RACE.yaml | 2 +- collections/GG_Google_Mobility_Data_trilateral.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/collections/GG_Google_Mobility_Data_RACE.yaml b/collections/GG_Google_Mobility_Data_RACE.yaml index 336fa0e8..2c71adff 100644 --- a/collections/GG_Google_Mobility_Data_RACE.yaml +++ b/collections/GG_Google_Mobility_Data_RACE.yaml @@ -19,4 +19,4 @@ Resources: Database: eodash CollectionId: Mobility_data AdditionalQueryString: 'aoi_id=in.("AT","BA","BE","BG","CH","CZ","DE","DK","EE","EG","ES","FI","FR","GB","GR","HR","HU","IE","IT","LT","LU","LV","MK","MT","NL","NO","PL","PT","RO","RS","RU","SE","SI","SK","TR")' - IdKey: 'country' + IdKey: 'aoi_id' diff --git a/collections/GG_Google_Mobility_Data_trilateral.yaml b/collections/GG_Google_Mobility_Data_trilateral.yaml index b7318f8f..5011b61e 100644 --- a/collections/GG_Google_Mobility_Data_trilateral.yaml +++ b/collections/GG_Google_Mobility_Data_trilateral.yaml @@ -18,4 +18,4 @@ Resources: Name: GeoDB Database: eodash CollectionId: Mobility_data - IdKey: 'country' + IdKey: 'aoi_id'