diff --git a/wis2box-management/wis2box/auth.py b/wis2box-management/wis2box/auth.py index c22edfebb..2451dd6cc 100644 --- a/wis2box-management/wis2box/auth.py +++ b/wis2box-management/wis2box/auth.py @@ -25,22 +25,21 @@ from secrets import token_hex from wis2box import cli_helpers -from wis2box.topic_hierarchy import validate_and_load from wis2box.env import AUTH_URL LOGGER = logging.getLogger(__name__) -def create_token(topic: str, token: str) -> bool: +def create_token(path: str, token: str) -> bool: """ Creates a token with access control - :param topic: `str` topic hierarchy + :param path: `str` path :param token: `str` authentication token :returns: `bool` of result """ - data = {'topic': topic, 'token': token} + data = {'path': path, 'token': token} r = requests.post(f'{AUTH_URL}/add_token', data=data) LOGGER.info(r.json().get('description')) @@ -48,16 +47,16 @@ def create_token(topic: str, token: str) -> bool: return r.ok -def delete_token(topic: str, token: str = '') -> bool: +def delete_token(path: str, token: str = '') -> bool: """ Creates a token with access control - :param topic: `str` topic hierarchy + :param path: `str` path :param token: `str` authentication token :returns: `bool` of result """ - data = {'topic': topic} + data = {'path': path} if token != '': # Delete all tokens for a given th @@ -69,37 +68,35 @@ def delete_token(topic: str, token: str = '') -> bool: return r.ok -def is_resource_open(topic: str) -> bool: +def is_resource_open(path: str) -> bool: """ - Checks if topic has access control configured + Checks if path has access control configured - :param topic: `str` topic hierarchy + :param path: `str` path :returns: `bool` of result """ - headers = {'X-Original-URI': topic} + headers = {'X-Original-URI': path} r = requests.get(f'{AUTH_URL}/authorize', headers=headers) return r.ok -def is_token_authorized(topic: str, token: str) -> bool: +def is_token_authorized(path: str, token: str) -> bool: """ - Checks if token is authorized to access a topic + Checks if token is authorized to access a path - :param topic: `str` topic hierarchy + :param path: `str` path :param token: `str` authentication token :returns: `bool` of result """ headers = { - 'X-Original-URI': topic, + 'X-Original-URI': path, 'Authorization': f'Bearer {token}', } - r = requests.get(f'{AUTH_URL}/authorize', headers=headers) - return r.ok @@ -111,11 +108,10 @@ def auth(): @click.command() @click.pass_context -@cli_helpers.OPTION_TOPIC_HIERARCHY -def is_restricted_topic(ctx, topic_hierarchy): - """Check if topic has access control""" - th, _ = validate_and_load(topic_hierarchy) - click.echo(not is_resource_open(th.dotpath)) +@cli_helpers.OPTION_DATASET +def is_restricted_dataset(ctx, metadata_id): + """Check if dataset has access control""" + click.echo(not is_resource_open(metadata_id)) @click.command() @@ -128,12 +124,11 @@ def is_restricted_path(ctx, path): @click.command() @click.pass_context -@cli_helpers.OPTION_TOPIC_HIERARCHY +@cli_helpers.OPTION_DATASET @click.argument('token') -def has_access_topic(ctx, topic_hierarchy, token): - """Check if a token has access to a topic""" - th, _ = validate_and_load(topic_hierarchy) - click.echo(is_token_authorized(th.dotpath, token)) +def has_access_dataset(ctx, metadata_id, token): + """Check if a token has access to a dataset""" + click.echo(is_token_authorized(metadata_id, token)) @click.command() @@ -147,20 +142,19 @@ def has_access_path(ctx, path, token): @click.command() @click.pass_context -@cli_helpers.OPTION_TOPIC_HIERARCHY +@cli_helpers.OPTION_DATASET @click.option('--path', '-p') @click.option('--yes', '-y', default=False, is_flag=True, help='Automatic yes') @click.argument('token', required=False) -def add_token(ctx, topic_hierarchy, path, yes, token): - """Add access token for a topic""" +def add_token(ctx, metadata_id, path, yes, token): + """Add access token for a path or dataset""" - if topic_hierarchy is not None: - th, _ = validate_and_load(topic_hierarchy) - topic = th.dotpath + if metadata_id is not None: + path = metadata_id elif path is not None: - topic = path + path = path else: - raise click.ClickException('Missing path or topic hierarchy') + raise click.ClickException('Missing path or metadata_id') token = token_hex(32) if token is None else token if yes: @@ -168,33 +162,33 @@ def add_token(ctx, topic_hierarchy, path, yes, token): elif not click.confirm(f'Continue with token: {token}', prompt_suffix='?'): return - if create_token(topic, token): + if create_token(path, token): click.echo('Token successfully created') @click.command() @click.pass_context -@cli_helpers.OPTION_TOPIC_HIERARCHY +@cli_helpers.OPTION_DATASET @click.option('--path', '-p') @click.argument('token', required=False, nargs=-1) -def remove_token(ctx, topic_hierarchy, path, token): - """Delete one to many tokens for a topic""" +def remove_token(ctx, metadata_id, path, token): + """Delete one to many tokens for a dataset""" + - if topic_hierarchy is not None: - th, _ = validate_and_load(topic_hierarchy) - topic = th.dotpath + if metadata_id is not None: + path = metadata_id elif path is not None: - topic = path + path = path else: - raise click.ClickException('Missing path or topic hierarchy') + raise click.ClickException('Missing path or metadata_id') - if delete_token(topic, token): + if delete_token(path, token): click.echo('Token successfully deleted') auth.add_command(add_token) auth.add_command(remove_token) -auth.add_command(has_access_topic) +auth.add_command(has_access_dataset) auth.add_command(has_access_path) -auth.add_command(is_restricted_topic) +auth.add_command(is_restricted_dataset) auth.add_command(is_restricted_path) diff --git a/wis2box-management/wis2box/data/base.py b/wis2box-management/wis2box/data/base.py index fb3961ff1..e8ed1d903 100644 --- a/wis2box-management/wis2box/data/base.py +++ b/wis2box-management/wis2box/data/base.py @@ -25,11 +25,12 @@ import re from typing import Iterator, Union -from wis2box.env import (STORAGE_INCOMING, STORAGE_PUBLIC, +from wis2box.env import (STORAGE_PUBLIC, STORAGE_SOURCE, BROKER_PUBLIC, DOCKER_BROKER) from wis2box.storage import exists, get_data, put_data -from wis2box.topic_hierarchy import TopicHierarchy +from wis2box.dataset import Dataset + from wis2box.plugin import load_plugin, PLUGINS from wis2box.pubsub.message import WISNotificationMessage @@ -52,7 +53,7 @@ def __init__(self, defs: dict) -> None: LOGGER.debug('Parsing resource mappings') self.filename = None self.incoming_filepath = None - self.topic_hierarchy = TopicHierarchy(defs['topic_hierarchy']) + self.dataset = Dataset(defs['dataset']) self.template = defs.get('template', None) self.file_filter = defs.get('pattern', '.*') self.enable_notification = defs.get('notify', False) @@ -91,10 +92,6 @@ def setup_discovery_metadata(self, discovery_metadata: dict) -> None: """ self.discovery_metadata = discovery_metadata - - self.topic_hierarchy = TopicHierarchy( - discovery_metadata['metadata']['identifier']) - self.country = discovery_metadata['wis2box']['country'] self.centre_id = discovery_metadata['wis2box']['centre_id'] @@ -146,13 +143,13 @@ def notify(self, identifier: str, storage_path: str, LOGGER.info('Publishing WISNotificationMessage to public broker') LOGGER.debug(f'Prepare message for: {storage_path}') - topic = f'origin/a/wis2/{self.topic_hierarchy.dirpath}' - data_id = topic.replace('origin/a/wis2/', '') + topic = self.dataset.topic_hierarchy + metadata_id = self.dataset.metadata_id operation = 'create' if is_update is False else 'update' wis_message = WISNotificationMessage( - identifier, data_id, storage_path, datetime_, geometry, + identifier, metadata_id, storage_path, datetime_, geometry, wigos_station_identifier, operation) # load plugin for public broker @@ -299,17 +296,6 @@ def files(self) -> Iterator[str]: yield f'{STORAGE_PUBLIC}/{rfp}/{identifier}.{format_}' - @property - def directories(self): - """Dataset directories""" - - dirpath = self.topic_hierarchy.dirpath - - return { - 'incoming': f'{STORAGE_INCOMING}/{dirpath}', - 'public': f'{STORAGE_PUBLIC}/{dirpath}' - } - def get_public_filepath(self): """Public filepath""" diff --git a/wis2box-management/wis2box/handler.py b/wis2box-management/wis2box/handler.py index e39dc358d..53f11f845 100644 --- a/wis2box-management/wis2box/handler.py +++ b/wis2box-management/wis2box/handler.py @@ -25,7 +25,7 @@ from wis2box.api import upsert_collection_item from wis2box.storage import get_data -from wis2box.topic_hierarchy import validate_and_load +from wis2box.dataset import validate_and_load from wis2box.plugin import load_plugin from wis2box.plugin import PLUGINS @@ -37,7 +37,7 @@ class Handler: def __init__(self, filepath: str, - topic_hierarchy: str = None, + metadata_id: str = None, data_mappings: dict = None) -> None: self.filepath = filepath self.plugins = () @@ -56,8 +56,7 @@ def __init__(self, filepath: str, if self.filepath.startswith('http'): self.input_bytes = get_data(self.filepath) - if topic_hierarchy is not None: - th = topic_hierarchy + if metadata_id is not None: fuzzy = False else: th = self.filepath @@ -68,10 +67,10 @@ def __init__(self, filepath: str, raise NotHandledError(msg) try: - self.topic_hierarchy, self.plugins = validate_and_load( + self.metadata_id, self.plugins = validate_and_load( th, data_mappings, self.filetype, fuzzy=fuzzy) except Exception as err: - msg = f'Topic Hierarchy validation error: {err}' + msg = f'Path validation error: {err}' # errors in public storage are not handled if STORAGE_PUBLIC in self.filepath: raise NotHandledError(msg) diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 3cceae2d2..43ada8dc6 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -178,8 +178,11 @@ def publish_broker_message(record: dict, storage_path: str, topic = f'origin/a/wis2/{centre_id.lower()}/metadata' # noqa datetime_ = datetime.strptime(record['properties']['created'], '%Y-%m-%dT%H:%M:%SZ') # noqa - wis_message = WISNotificationMessage(record['id'], topic, storage_path, - datetime_, record['geometry']).dumps() + wis_message = WISNotificationMessage(identifier=record['id'], + metadata_id=None, + filepath=storage_path, + datetime_=datetime_, + geometry=record['geometry']).dumps() # load plugin for plugin-broker defs = { diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index 9e51aff7b..e8788ab31 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -53,7 +53,7 @@ class PubSubMessage: Generic message class """ - def __init__(self, type_: str, identifier: str, topic: str, filepath: str, + def __init__(self, type_: str, identifier: str, metadata_id: str, filepath: str, datetime_: datetime, geometry: dict = None, wigos_station_identifier: str = None) -> None: """ @@ -61,7 +61,7 @@ def __init__(self, type_: str, identifier: str, topic: str, filepath: str, :param type_: message type :param identifier: identifier - :param topic: topic + :param metadata_id: metadata_id :param filepath: `Path` of file :param datetime_: `datetime` object of temporal aspect of data :param geometry: `dict` of GeoJSON geometry object @@ -128,16 +128,16 @@ def _generate_checksum(self, bytes, algorithm: SecureHashAlgorithms) -> str: # class WISNotificationMessage(PubSubMessage): - def __init__(self, identifier: str, topic: str, filepath: str, + def __init__(self, identifier: str, metadata_id: str, filepath: str, datetime_: str, geometry=None, wigos_station_identifier=None, operation: str = 'create') -> None: super().__init__('wis2-notification-message', identifier, - topic, filepath, datetime_, geometry) + metadata_id, filepath, datetime_, geometry) - data_id = f'{topic}/{self.identifier}'.replace('origin/a/wis2/', '') + data_id = f'{metadata_id}/{self.identifier}'.replace('origin/a/wis2/', '') - if '/metadata' in topic: + if '/metadata' in metadata_id: mimetype = 'application/geo+json' else: suffix = self.filepath.split('.')[-1] diff --git a/wis2box-management/wis2box/topic_hierarchy.py b/wis2box-management/wis2box/topic_hierarchy.py index 9501bad3b..58cbb29bd 100644 --- a/wis2box-management/wis2box/topic_hierarchy.py +++ b/wis2box-management/wis2box/topic_hierarchy.py @@ -67,36 +67,36 @@ def is_valid(self) -> bool: return True -def validate_and_load(topic_hierarchy: str, +def validate_and_load(path: str, data_mappings: dict = None, file_type: str = None, fuzzy: bool = False ) -> Tuple[TopicHierarchy, Tuple[Any]]: """ - Validate topic hierarchy and load plugins + Validate path and load plugins - :param topic_hierarchy: `str` of topic hierarchy path + :param path: `str` of path :param data_mappings: `dict` of data mappings :param file_type: `str` the type of file we are processing, e.g. csv, bufr, xml # noqa - :param fuzzy: `bool` of whether to do fuzzy matching of topic hierarchy + :param fuzzy: `bool` of whether to do fuzzy matching of path (e.g. "*foo.bar.baz*). Defaults to `False` (i.e. "foo.bar.baz") - :returns: tuple of `wis2box.topic_hierarchy.TopicHierarchy` and + :returns: tuple of `wis2box.dataset.Dataset` and list of plugins objects """ - LOGGER.debug(f'Validating topic hierarchy: {topic_hierarchy}') + LOGGER.debug(f'Validating path: {path}') LOGGER.debug(f'Data mappings {data_mappings}') if not data_mappings: msg = 'Data mappings are empty. Fetching' data_mappings = get_data_mappings() - th = TopicHierarchy(topic_hierarchy) + dataset = Dataset(path) found = False - if not th.is_valid(): + if not dataset.is_valid(): msg = 'Invalid topic hierarchy' raise ValueError(msg)