remove dotpath in test-files, update data_id, don't use topic_hierarc…

…hy for data_name
wmo-im · May 23, 2024 · c60b2aa · c60b2aa
1 parent 87f2977
commit c60b2aa
Show file tree

Hide file tree

Showing 13 changed files with 16 additions and 17 deletions.
diff --git a/tests/data/metadata/discovery/cd-surface-weather-observations.yml b/tests/data/metadata/discovery/cd-surface-weather-observations.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P180D
-    topic_hierarchy: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: cd-brazza_met_centre/data/core/weather/surface-based-observations/synop
     country: cog
     centre_id: cd-brazza_met_centre
     data_mappings:

diff --git a/tests/data/metadata/discovery/cn-grapes-geps-global.yml b/tests/data/metadata/discovery/cn-grapes-geps-global.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global
+    topic_hierarchy: cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global
     country: chn
     centre_id: cn-cma
     data_mappings:

diff --git a/tests/data/metadata/discovery/dz-surface-weather-observations.yml b/tests/data/metadata/discovery/dz-surface-weather-observations.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: dz-alger_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: dz-alger_met_centre/data/core/weather/surface-based-observations/synop
     country: dza
     centre_id: dz-alger_met_centre
     data_mappings:

diff --git a/tests/data/metadata/discovery/int-wmo-test-buoy.yml b/tests/data/metadata/discovery/int-wmo-test-buoy.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.buoy
+    topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/buoy
     country: int
     centre_id: int-wmo-test
     data_mappings:

diff --git a/tests/data/metadata/discovery/int-wmo-test-ship.yml b/tests/data/metadata/discovery/int-wmo-test-ship.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.ship
+    topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship
     country: int
     centre_id: int-wmo-test
     data_mappings:

diff --git a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml b/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.wind_profiler
+    topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/wind_profiler
     country: int
     centre_id: int-wmo-test
     data_mappings:

diff --git a/tests/data/metadata/discovery/it-surface-weather-observations.yml b/tests/data/metadata/discovery/it-surface-weather-observations.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: it-roma_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: it-roma_met_centre/data/core/weather/surface-based-observations/synop
     country: ita
     centre_id: it-roma_met_centre
     data_mappings:

diff --git a/tests/data/metadata/discovery/mw-surface-weather-observations.yml b/tests/data/metadata/discovery/mw-surface-weather-observations.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: mw-mw_met_centre/data/core/weather/surface-based-observations/synop
     country: mwi
     centre_id: mw-mw_met_centre
     data_mappings:

diff --git a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml
@@ -1,6 +1,6 @@
 wis2box:
     retention: P30D
-    topic_hierarchy: ro-rnimh.data.core.weather.surface-based-observations.synop
+    topic_hierarchy: ro-rnimh/data/core/weather/surface-based-observations/synop
     country: rou
     centre_id: ro-rnimh
     data_mappings:

diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py
@@ -98,10 +98,7 @@ def setup_collection(meta: dict = {}) -> bool:
         LOGGER.error(f'Invalid configuration: {meta}')
         return False
 
-    if 'topic_hierarchy' in meta:
-        data_name = meta['topic_hierarchy']
-    else:
-        data_name = meta['id']
+    data_name = meta['id']
 
     backend = load_backend()
     if not backend.has_collection(data_name):

diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py
@@ -150,7 +150,7 @@ def notify(self, identifier: str, storage_path: str,
         operation = 'create' if is_update is False else 'update'
 
         wis_message = WISNotificationMessage(
-            identifier, metadata_id, storage_path, datetime_, geometry,
+            f'{metadata_id}/{identifier}', metadata_id, storage_path, datetime_, geometry,
             wigos_station_identifier, operation)
 
         # load plugin for public broker

diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py
@@ -178,7 +178,7 @@ def publish_broker_message(record: dict, storage_path: str,
     topic = f'origin/a/wis2/{centre_id.lower()}/metadata'  # noqa
 
     datetime_ = datetime.strptime(record['properties']['created'], '%Y-%m-%dT%H:%M:%SZ')  # noqa
-    wis_message = WISNotificationMessage(identifier=record['id'],
+    wis_message = WISNotificationMessage(identifier=f"{centre_id.lower()}/metadata/{record['id']}",
                                          metadata_id=None,
                                          filepath=storage_path,
                                          datetime_=datetime_,

diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py
@@ -132,7 +132,7 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str,
         super().__init__('wis2-notification-message', identifier,
                          filepath, datetime_, geometry)
 
-        data_id = f'{metadata_id}/{self.identifier}'
+        data_id = f'{self.identifier}'
 
         if '/metadata' in filepath:
             mimetype = 'application/geo+json'
@@ -172,7 +172,6 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str,
             'geometry': self.geometry,
             'properties': {
                 'data_id': data_id,
-                'metadata_id': metadata_id,
                 'datetime': self.datetime,
                 'pubtime': self.publish_datetime,
                 'integrity': {
@@ -184,6 +183,9 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str,
             'generated-by': f'wis2box {__version__}'
         }
 
+        if metadata_id is not None:
+            self.message['properties']['metadata_id'] = metadata_id
+
         if self.length < 4096:
             LOGGER.debug('Including data inline via properties.content')
             content_value = base64.b64encode(self.filebytes)