From c60b2aa3c7d0e65705517715486c4a523cf709d5 Mon Sep 17 00:00:00 2001 From: Maaike Date: Thu, 23 May 2024 14:22:54 +0200 Subject: [PATCH] remove dotpath in test-files, update data_id, don't use topic_hierarchy for data_name --- .../metadata/discovery/cd-surface-weather-observations.yml | 2 +- tests/data/metadata/discovery/cn-grapes-geps-global.yml | 2 +- .../metadata/discovery/dz-surface-weather-observations.yml | 2 +- tests/data/metadata/discovery/int-wmo-test-buoy.yml | 2 +- tests/data/metadata/discovery/int-wmo-test-ship.yml | 2 +- .../data/metadata/discovery/int-wmo-test-wind_profiler.yml | 2 +- .../metadata/discovery/it-surface-weather-observations.yml | 2 +- .../metadata/discovery/mw-surface-weather-observations.yml | 2 +- .../metadata/discovery/ro-synoptic-weather-observations.yml | 2 +- wis2box-management/wis2box/api/__init__.py | 5 +---- wis2box-management/wis2box/data/base.py | 2 +- wis2box-management/wis2box/metadata/discovery.py | 2 +- wis2box-management/wis2box/pubsub/message.py | 6 ++++-- 13 files changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/data/metadata/discovery/cd-surface-weather-observations.yml b/tests/data/metadata/discovery/cd-surface-weather-observations.yml index 0a0a9d8c4..76dcda288 100644 --- a/tests/data/metadata/discovery/cd-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/cd-surface-weather-observations.yml @@ -1,6 +1,6 @@ wis2box: retention: P180D - topic_hierarchy: cd-brazza_met_centre.data.core.weather.surface-based-observations.synop + topic_hierarchy: cd-brazza_met_centre/data/core/weather/surface-based-observations/synop country: cog centre_id: cd-brazza_met_centre data_mappings: diff --git a/tests/data/metadata/discovery/cn-grapes-geps-global.yml b/tests/data/metadata/discovery/cn-grapes-geps-global.yml index 180854e4e..035270e82 100644 --- a/tests/data/metadata/discovery/cn-grapes-geps-global.yml +++ b/tests/data/metadata/discovery/cn-grapes-geps-global.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: cn-cma.data.core.weather.prediction.forecast.medium-range.probabilistic.global + topic_hierarchy: cn-cma/data/core/weather/prediction/forecast/medium-range/probabilistic/global country: chn centre_id: cn-cma data_mappings: diff --git a/tests/data/metadata/discovery/dz-surface-weather-observations.yml b/tests/data/metadata/discovery/dz-surface-weather-observations.yml index 1d0b2a407..a850c49c7 100644 --- a/tests/data/metadata/discovery/dz-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/dz-surface-weather-observations.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: dz-alger_met_centre.data.core.weather.surface-based-observations.synop + topic_hierarchy: dz-alger_met_centre/data/core/weather/surface-based-observations/synop country: dza centre_id: dz-alger_met_centre data_mappings: diff --git a/tests/data/metadata/discovery/int-wmo-test-buoy.yml b/tests/data/metadata/discovery/int-wmo-test-buoy.yml index 102cb95fb..e439731de 100644 --- a/tests/data/metadata/discovery/int-wmo-test-buoy.yml +++ b/tests/data/metadata/discovery/int-wmo-test-buoy.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.buoy + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/buoy country: int centre_id: int-wmo-test data_mappings: diff --git a/tests/data/metadata/discovery/int-wmo-test-ship.yml b/tests/data/metadata/discovery/int-wmo-test-ship.yml index 52417815f..56a37695e 100644 --- a/tests/data/metadata/discovery/int-wmo-test-ship.yml +++ b/tests/data/metadata/discovery/int-wmo-test-ship.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.ship + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/ship country: int centre_id: int-wmo-test data_mappings: diff --git a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml b/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml index eec608cb6..e6eeacd36 100644 --- a/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml +++ b/tests/data/metadata/discovery/int-wmo-test-wind_profiler.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: int-wmo-test.data.core.weather.surface-based-observations.wind_profiler + topic_hierarchy: int-wmo-test/data/core/weather/surface-based-observations/wind_profiler country: int centre_id: int-wmo-test data_mappings: diff --git a/tests/data/metadata/discovery/it-surface-weather-observations.yml b/tests/data/metadata/discovery/it-surface-weather-observations.yml index a64322c84..14e2c1a14 100644 --- a/tests/data/metadata/discovery/it-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/it-surface-weather-observations.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: it-roma_met_centre.data.core.weather.surface-based-observations.synop + topic_hierarchy: it-roma_met_centre/data/core/weather/surface-based-observations/synop country: ita centre_id: it-roma_met_centre data_mappings: diff --git a/tests/data/metadata/discovery/mw-surface-weather-observations.yml b/tests/data/metadata/discovery/mw-surface-weather-observations.yml index bcb14c489..bfd80d255 100644 --- a/tests/data/metadata/discovery/mw-surface-weather-observations.yml +++ b/tests/data/metadata/discovery/mw-surface-weather-observations.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: mw-mw_met_centre.data.core.weather.surface-based-observations.synop + topic_hierarchy: mw-mw_met_centre/data/core/weather/surface-based-observations/synop country: mwi centre_id: mw-mw_met_centre data_mappings: diff --git a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml index 58fd2676e..e55d791bf 100644 --- a/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml +++ b/tests/data/metadata/discovery/ro-synoptic-weather-observations.yml @@ -1,6 +1,6 @@ wis2box: retention: P30D - topic_hierarchy: ro-rnimh.data.core.weather.surface-based-observations.synop + topic_hierarchy: ro-rnimh/data/core/weather/surface-based-observations/synop country: rou centre_id: ro-rnimh data_mappings: diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py index 349b8d839..c96601d4f 100644 --- a/wis2box-management/wis2box/api/__init__.py +++ b/wis2box-management/wis2box/api/__init__.py @@ -98,10 +98,7 @@ def setup_collection(meta: dict = {}) -> bool: LOGGER.error(f'Invalid configuration: {meta}') return False - if 'topic_hierarchy' in meta: - data_name = meta['topic_hierarchy'] - else: - data_name = meta['id'] + data_name = meta['id'] backend = load_backend() if not backend.has_collection(data_name): diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index 959726583..d1c292df8 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -150,7 +150,7 @@ def notify(self, identifier: str, storage_path: str, operation = 'create' if is_update is False else 'update' wis_message = WISNotificationMessage( - identifier, metadata_id, storage_path, datetime_, geometry, + f'{metadata_id}/{identifier}', metadata_id, storage_path, datetime_, geometry, wigos_station_identifier, operation) # load plugin for public broker diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 43ada8dc6..46e50575d 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -178,7 +178,7 @@ def publish_broker_message(record: dict, storage_path: str, topic = f'origin/a/wis2/{centre_id.lower()}/metadata' # noqa datetime_ = datetime.strptime(record['properties']['created'], '%Y-%m-%dT%H:%M:%SZ') # noqa - wis_message = WISNotificationMessage(identifier=record['id'], + wis_message = WISNotificationMessage(identifier=f"{centre_id.lower()}/metadata/{record['id']}", metadata_id=None, filepath=storage_path, datetime_=datetime_, diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index e86e9252e..d036a630e 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -132,7 +132,7 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, super().__init__('wis2-notification-message', identifier, filepath, datetime_, geometry) - data_id = f'{metadata_id}/{self.identifier}' + data_id = f'{self.identifier}' if '/metadata' in filepath: mimetype = 'application/geo+json' @@ -172,7 +172,6 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, 'geometry': self.geometry, 'properties': { 'data_id': data_id, - 'metadata_id': metadata_id, 'datetime': self.datetime, 'pubtime': self.publish_datetime, 'integrity': { @@ -184,6 +183,9 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, 'generated-by': f'wis2box {__version__}' } + if metadata_id is not None: + self.message['properties']['metadata_id'] = metadata_id + if self.length < 4096: LOGGER.debug('Including data inline via properties.content') content_value = base64.b64encode(self.filebytes)