From 3996b92d59404cdd9993cd51eb7a3c15eebf4236 Mon Sep 17 00:00:00 2001 From: Maaike Date: Mon, 2 Dec 2024 10:12:20 +0100 Subject: [PATCH] use get_plugins to decide whether or not to add oafeat-link --- wis2box-management/wis2box/api/__init__.py | 28 ++-------- wis2box-management/wis2box/data_mappings.py | 22 ++++++++ .../wis2box/metadata/discovery.py | 53 +++++++++++-------- 3 files changed, 57 insertions(+), 46 deletions(-) diff --git a/wis2box-management/wis2box/api/__init__.py b/wis2box-management/wis2box/api/__init__.py index 5db7679ad..51cf51e26 100644 --- a/wis2box-management/wis2box/api/__init__.py +++ b/wis2box-management/wis2box/api/__init__.py @@ -30,6 +30,8 @@ from wis2box import cli_helpers from wis2box.api.backend import load_backend from wis2box.api.config import load_config +from wis2box.data_mappings import get_plugins + from wis2box.env import (DOCKER_API_URL, API_URL) LOGGER = logging.getLogger(__name__) @@ -232,28 +234,6 @@ def delete_collections_by_retention(days: int) -> None: backend.delete_collections_by_retention(days) -def get_plugins(record: dict) -> list: - """ - Get plugins from record - - :param record: `dict` of record - - :returns: `list` of plugins - """ - - plugins = [] - - try: - dm = record['wis2box']['data_mappings'] - for filetype in dm['plugins'].keys(): - for p in dm['plugins'][filetype]: - plugins.append(p['plugin']) - except Exception as e: - LOGGER.info(f"No plugins found for record-id={record['id']} : {e}") - - return plugins - - @click.group() def api(): """API management""" @@ -285,7 +265,9 @@ def setup(ctx, verbosity): metadata_id = record['id'] plugins = get_plugins(record) LOGGER.info(f'Plugins used by {metadata_id}: {plugins}') - if 'wis2box.data.bufr2geojson.ObservationDataBUFR2GeoJSON' not in plugins: # noqa + # check if any plugin-names contains 2geojson + has_2geojson = any('2geojson' in plugin for plugin in plugins) + if has_2geojson is False: continue if metadata_id not in api_collections: click.echo(f'Adding data-collection for: {metadata_id}') diff --git a/wis2box-management/wis2box/data_mappings.py b/wis2box-management/wis2box/data_mappings.py index b81b131a2..cc5cdd220 100644 --- a/wis2box-management/wis2box/data_mappings.py +++ b/wis2box-management/wis2box/data_mappings.py @@ -31,6 +31,28 @@ LOGGER = logging.getLogger(__name__) +def get_plugins(record: dict) -> list: + """ + Get plugins from record + + :param record: `dict` of record + + :returns: `list` of plugins + """ + + plugins = [] + + try: + dm = record['wis2box']['data_mappings'] + for filetype in dm['plugins'].keys(): + for p in dm['plugins'][filetype]: + plugins.append(p['plugin']) + except Exception as e: + LOGGER.info(f"No plugins found for record-id={record['id']} : {e}") + + return plugins + + def refresh_data_mappings(): # load plugin for local broker and publish refresh request defs_local = { diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 332275c72..d32c69a3a 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -35,7 +35,8 @@ from wis2box import cli_helpers from wis2box.api import (delete_collection_item, remove_collection, setup_collection, upsert_collection_item) -from wis2box.data_mappings import refresh_data_mappings +from wis2box.data_mappings import refresh_data_mappings, get_plugins + from wis2box.env import (API_URL, BROKER_PUBLIC, DOCKER_API_URL, STORAGE_PUBLIC, STORAGE_SOURCE, URL) from wis2box.metadata.base import BaseMetadata @@ -76,17 +77,7 @@ def generate(self, mcf: dict) -> str: if md['identification']['extents']['temporal'][0].get('begin', 'BEGIN_DATE') is None: # noqa today = date.today().strftime('%Y-%m-%d') md['identification']['extents']['temporal'][0]['begin'] = today - - LOGGER.debug('Adding distribution links') - oafeat_link, mqp_link, canonical_link = self.get_distribution_links( - identifier, mqtt_topic, format_='mcf') - - md['distribution'] = { - 'oafeat': oafeat_link, - 'mqtt': mqp_link, - 'canonical': canonical_link - } - + LOGGER.debug('Adding data policy') md['identification']['wmo_data_policy'] = mqtt_topic.split('/')[5] @@ -95,6 +86,13 @@ def generate(self, mcf: dict) -> str: record['properties']['wmo:topicHierarchy'] = mqtt_topic record['wis2box'] = mcf['wis2box'] + LOGGER.debug('Adding distribution links') + distribution_links = self.get_distribution_links( + record, + format_='wcmp2') + # update links, do not extend or we get duplicates + record['links'] = distribution_links + if record['properties']['contacts'][0].get('organization') is None: record['properties']['contacts'][0]['organization'] = record['properties']['contacts'][0].pop('name', "NOTSET") # noqa @@ -118,26 +116,35 @@ def generate(self, mcf: dict) -> str: return record - def get_distribution_links(self, identifier: str, topic: str, + def get_distribution_links(self, + record: dict, format_='mcf') -> list: """ Generates distribution links - :param identifier: `str` of metadata identifier - :param topic: `str` of associated topic + :param record: `dict` of discovery metadata record :param format_: `str` of format (`mcf` or `wcmp2`) :returns: `list` of distribution links """ LOGGER.debug('Adding distribution links') - oafeat_link = { - 'href': f"{API_URL}/collections/{identifier}?f=json", - 'type': 'application/json', - 'name': identifier, - 'description': identifier, - 'rel': 'collection' - } + + identifier = record['id'] + topic = record['properties']['wmo:topicHierarchy'] + + oafeat_link = None + plugins = get_plugins(record) + # check if any plugin-names contains 2geojson + has_2geojson = any('2geojson' in plugin for plugin in plugins) + if has_2geojson: + oafeat_link = { + 'href': f"{API_URL}/collections/{identifier}?f=json", + 'type': 'application/json', + 'name': identifier, + 'description': f'Observations in json format for {identifier}', + 'rel': 'collection' + } mqp_link = { 'href': get_broker_public_endpoint(), @@ -240,7 +247,7 @@ def publish_discovery_metadata(metadata: Union[dict, str]): record = metadata dm = DiscoveryMetadata() distribution_links = dm.get_distribution_links( - record['id'], record['properties']['wmo:topicHierarchy'], + record, format_='wcmp2') # update links, do not extend or we get duplicates record['links'] = distribution_links