Integration of new endpoint type and additional instance (#41)

* feat: implementing gtif indicator collections; adding xyz visualization for vectortiles

* fix: wrong identifier

* chore: changing title

* feat: adding extra fields needed for client visualization of data to xyz vector link

* Update README.md

README.md

@@ -1 +1,25 @@
-# eodash-catalog
+# eodash-catalog
+
+This repository allows generation of STAC catalogs based on the configuration yaml files provided in the repository.
+The configuration options are described in the [Wiki](../../wiki).
+
+The generation of the catalog runs automatically through github actions when pushing/merging to main branch.
+The generated STAC catalogs are deployed through github pages.
+
+A preview of the catalogs can be seen using the [Stac Browser](https://radiantearth.github.io/stac-browser/#/)
+
+Here are preloaded preview links for the catalogs available through gh-pages:
+* [RACE Instance](https://radiantearth.github.io/stac-browser/#/external/eurodatacube.github.io/eodash-catalog/RACE/catalog.json)
+* [Trilateral Instance](https://radiantearth.github.io/stac-browser/#/external/eurodatacube.github.io/eodash-catalog/trilateral/catalog.json)
+* [GTIF Instance](https://radiantearth.github.io/stac-browser/#/external/eurodatacube.github.io/eodash-catalog/GTIF/catalog.json)
+
+In order to run the catalog generation locally you can clone the repository and ideally setup a virtual environment or similar using the requirements.txt contained in the `generators` folder.
+If you are using Visual Studio Code you can use the `Python: Create environment...` tool (when pressing Ctrl+Shift+P).
+Use of python >=3.10 is recommended.
+
+Once venv is setup, make sure it is active and run the generator with `python generate_indicators.py`. 
+Currently there is only one optional flag `-vd` that can be used to activate validation of the catalog (Still WIP).
+
+The catalogs are saved in the `build` folder. If you want to test the generated catalog locally (either in the Stac Browser or with the eodash client) we recommend using npm [http-server](https://www.npmjs.com/package/http-server), especially to avoid possible CORS issues you can run it for example with following command:
+`npx http-server -p 8000 --cors` 
+when located in the build folder.

catalogs/GTIF.yaml

@@ -0,0 +1,11 @@
+id: "GTIF"
+title: "GTIF"
+description: "Green Transition Information Factory"
+endpoint: "https://eurodatacube.github.io/eodash-catalog/GTIF/"
+assets_endpoint: "https://raw.githubusercontent.com/eurodatacube/eodash-assets/main/collections"
+collections:
+  # - E12c_truck_detections_motorways
+  # - E12c_truck_detections_primary_roads
+  # - AQ1_aggregated_mobility_data
+  - AQA_health_risk_index
+

collections/AQ1_aggregated_mobility_data.yaml

@@ -0,0 +1,37 @@
+Name: aggregated_mobility_data
+Title: Correlation explorer
+EodashIdentifier: AQ1
+Subtitle: Aggregated mobility data
+Description: AQ1_aggregated_mobility_data/AQ1.md
+Themes:
+  - economy
+Tags:
+  - placeholder
+Satellite:
+  - placeholder
+Sensor:
+  - MODIS
+Agency:
+  - ESA
+Provider:
+  - SISTEMA
+Dates: AQ1_aggregated_mobility_data/dates.json # Allows to bypass retrieval of dates
+Resources:
+  - EndPoint: https://xcube-geodb.brockmann-consult.de/geoserver/geodb_debd884d-92f9-4979-87b6-eadef1139394/gwc/service/tms/1.0.0/
+    DBEndpoint: https://xcube-geodb.brockmann-consult.de/gtif/f0ad1e25-98fa-4b82-9228-815ab24f5dd1/
+    Source: aggregated_trajs_model_satellite_v1
+    Name: GeoDB Vector Tiles
+    Instance: geodb_debd884d-92f9-4979-87b6-eadef1139394
+    Database: GTIF
+    CollectionId: grid_gtif_aggregated_data
+    TimeKey: timestamp
+    AdminZoneKey: satellite_id
+    Parameters:
+      - satellite_id
+      - satellite_values
+      - mean_value
+      - speed
+      - congestion_index
+      - n_trajectories
+      - motorized_count
+      - motorized_share

collections/AQA_health_risk_index.yaml

@@ -0,0 +1,32 @@
+Name: health_risk_index
+Title: Health Risk Index (ARI)
+EodashIdentifier: AQA
+Subtitle: Health Risk Index (ARI)
+Description: AQ1_health_risk_index/AQA.md
+Themes:
+  - economy
+Tags:
+  - placeholder
+Satellite:
+  - placeholder
+Sensor:
+  - MODIS
+Agency:
+  - ESA
+Provider:
+  - SISTEMA
+Resources:
+  - EndPoint: https://xcube-geodb.brockmann-consult.de/geoserver/geodb_debd884d-92f9-4979-87b6-eadef1139394/gwc/service/tms/1.0.0/
+    DBEndpoint: https://xcube-geodb.brockmann-consult.de/gtif/f0ad1e25-98fa-4b82-9228-815ab24f5dd1/
+    Source: air_quality_new_id
+    Name: GeoDB Vector Tiles
+    Instance: geodb_debd884d-92f9-4979-87b6-eadef1139394
+    Database: GTIF
+    CollectionId: AT_Gemeinden_3857
+    TimeKey: time
+    MatchKey: id_3
+    Parameters:
+      - pm10
+      - pm25
+      - ihr
+      - id_3

generators/generate_indicators.py

@@ -88,9 +88,45 @@ def process_collection_file(config, file_path, catalog):
                     handle_VEDA_endpoint(config, resource, data, catalog)
                 elif resource["Name"] == "WMS":
                     handle_WMS_endpoint(config, resource, data, catalog)
+                elif resource["Name"] == "GeoDB Vector Tiles":
+                    handle_GeoDB_Tiles_endpoint(config, resource, data, catalog)
                 else:
                     raise ValueError("Type of Resource is not supported")
 
+def handle_GeoDB_Tiles_endpoint(config, endpoint, data, catalog):
+    collection = create_collection(endpoint["CollectionId"], data, config)
+    if "Dates" in data:
+        pass
+    else:
+        select = "?select=%s"%endpoint["TimeKey"]
+        url = endpoint["DBEndpoint"] + endpoint["Database"] + "_%s"%endpoint["Source"] + select
+        response = json.loads(requests.get(url).text)
+        times = set([entry[endpoint["TimeKey"]] for entry in response])
+        if len(times) > 0:
+            # Create an item per time to allow visualization in stac clients
+            styles = None
+            if hasattr(endpoint, "Styles"):
+                styles = endpoint["Styles"]
+            collection = create_collection(data["Name"], data, config)
+            # TODO: For now we create global extent, we should be able to
+            # fetch the extent of the layer
+            for t in times:
+                item = Item(
+                    id = t,
+                    bbox=[-180.0, -90.0, 180.0, 90.0],
+                    properties={},
+                    geometry = None,
+                    datetime = parser.isoparse(t),
+                )
+                # add_visualization_info(item, data, endpoint, time=t, styles=styles)
+                link = collection.add_item(item)
+                link.extra_fields["datetime"] = t
+            collection.update_extent_from_items()
+            add_visualization_info(collection, data, endpoint, styles=styles)
+            add_collection_information(config, collection, data)
+            add_to_catalog(collection, catalog, endpoint, data)
+
+
 def handle_WMS_endpoint(config, endpoint, data, catalog):
     if endpoint["Type"] == "Time" or endpoint["Type"] == "OverwriteTimes":
 
@@ -424,6 +460,30 @@ def add_visualization_info(stac_object, data, endpoint, file_url=None, time=None
             )
         )
         pass
+    elif endpoint["Name"] == "GeoDB Vector Tiles":
+        #`${geoserverUrl}${config.layerName}@EPSG%3A${projString}@pbf/{z}/{x}/{-y}.pbf`,
+        # 'geodb_debd884d-92f9-4979-87b6-eadef1139394:GTIF_AT_Gemeinden_3857'
+        target_url = "%s%s:%s_%s@EPSG:3857@pbf/{z}/{x}/{-y}.pbf"%(
+            endpoint["EndPoint"],
+            endpoint["Instance"],
+            endpoint["Database"],
+            endpoint["CollectionId"],
+        )
+        stac_object.add_link(
+            Link(
+                rel="xyz",
+                target=target_url,
+                media_type="application/pbf",
+                title=data["Name"],
+                extra_fields={
+                    "description": data["Title"],
+                    "parameters": endpoint["Parameters"],
+                    "matchKey": endpoint["MatchKey"],
+                    "timeKey": endpoint["TimeKey"],
+                    "source" : endpoint["Source"],
+                }
+            )
+        )
     else:
         print("Visualization endpoint not supported")