WIP

wis2box-management/wis2box/auth.py

@@ -25,39 +25,38 @@
 from secrets import token_hex
 
 from wis2box import cli_helpers
-from wis2box.topic_hierarchy import validate_and_load
 from wis2box.env import AUTH_URL
 
 LOGGER = logging.getLogger(__name__)
 
 
-def create_token(topic: str, token: str) -> bool:
+def create_token(path: str, token: str) -> bool:
     """
     Creates a token with access control
 
-    :param topic: `str` topic hierarchy
+    :param path: `str` path
     :param token: `str` authentication token
 
     :returns: `bool` of result
     """
-    data = {'topic': topic, 'token': token}
+    data = {'path': path, 'token': token}
 
     r = requests.post(f'{AUTH_URL}/add_token', data=data)
     LOGGER.info(r.json().get('description'))
 
     return r.ok
 
 
-def delete_token(topic: str, token: str = '') -> bool:
+def delete_token(path: str, token: str = '') -> bool:
     """
     Creates a token with access control
 
-    :param topic: `str` topic hierarchy
+    :param path: `str` path
     :param token: `str` authentication token
 
     :returns: `bool` of result
     """
-    data = {'topic': topic}
+    data = {'path': path}
 
     if token != '':
         # Delete all tokens for a given th
@@ -69,37 +68,35 @@ def delete_token(topic: str, token: str = '') -> bool:
     return r.ok
 
 
-def is_resource_open(topic: str) -> bool:
+def is_resource_open(path: str) -> bool:
     """
-    Checks if topic has access control configured
+    Checks if path has access control configured
 
-    :param topic: `str` topic hierarchy
+    :param path: `str` path
 
     :returns: `bool` of result
     """
-    headers = {'X-Original-URI': topic}
+    headers = {'X-Original-URI': path}
 
     r = requests.get(f'{AUTH_URL}/authorize', headers=headers)
 
     return r.ok
 
 
-def is_token_authorized(topic: str, token: str) -> bool:
+def is_token_authorized(path: str, token: str) -> bool:
     """
-    Checks if token is authorized to access a topic
+    Checks if token is authorized to access a path
 
-    :param topic: `str` topic hierarchy
+    :param path: `str` path
     :param token: `str` authentication token
 
     :returns: `bool` of result
     """
     headers = {
-        'X-Original-URI': topic,
+        'X-Original-URI': path,
         'Authorization': f'Bearer {token}',
     }
-
     r = requests.get(f'{AUTH_URL}/authorize', headers=headers)
-
     return r.ok
 
 
@@ -111,11 +108,10 @@ def auth():
 
 @click.command()
 @click.pass_context
-@cli_helpers.OPTION_TOPIC_HIERARCHY
-def is_restricted_topic(ctx, topic_hierarchy):
-    """Check if topic has access control"""
-    th, _ = validate_and_load(topic_hierarchy)
-    click.echo(not is_resource_open(th.dotpath))
+@cli_helpers.OPTION_DATASET
+def is_restricted_dataset(ctx, metadata_id):
+    """Check if dataset has access control"""
+    click.echo(not is_resource_open(metadata_id))
 
 
 @click.command()
@@ -128,12 +124,11 @@ def is_restricted_path(ctx, path):
 
 @click.command()
 @click.pass_context
-@cli_helpers.OPTION_TOPIC_HIERARCHY
+@cli_helpers.OPTION_DATASET
 @click.argument('token')
-def has_access_topic(ctx, topic_hierarchy, token):
-    """Check if a token has access to a topic"""
-    th, _ = validate_and_load(topic_hierarchy)
-    click.echo(is_token_authorized(th.dotpath, token))
+def has_access_dataset(ctx, metadata_id, token):
+    """Check if a token has access to a dataset"""
+    click.echo(is_token_authorized(metadata_id, token))
 
 
 @click.command()
@@ -147,54 +142,53 @@ def has_access_path(ctx, path, token):
 
 @click.command()
 @click.pass_context
-@cli_helpers.OPTION_TOPIC_HIERARCHY
+@cli_helpers.OPTION_DATASET
 @click.option('--path', '-p')
 @click.option('--yes', '-y', default=False, is_flag=True, help='Automatic yes')
 @click.argument('token', required=False)
-def add_token(ctx, topic_hierarchy, path, yes, token):
-    """Add access token for a topic"""
+def add_token(ctx, metadata_id, path, yes, token):
+    """Add access token for a path or dataset"""
 
-    if topic_hierarchy is not None:
-        th, _ = validate_and_load(topic_hierarchy)
-        topic = th.dotpath
+    if metadata_id is not None:
+        path = metadata_id
     elif path is not None:
-        topic = path
+        path = path
     else:
-        raise click.ClickException('Missing path or topic hierarchy')
+        raise click.ClickException('Missing path or metadata_id')
 
     token = token_hex(32) if token is None else token
     if yes:
         click.echo(f'Using token: {token}')
     elif not click.confirm(f'Continue with token: {token}', prompt_suffix='?'):
         return
 
-    if create_token(topic, token):
+    if create_token(path, token):
         click.echo('Token successfully created')
 
 
 @click.command()
 @click.pass_context
-@cli_helpers.OPTION_TOPIC_HIERARCHY
+@cli_helpers.OPTION_DATASET
 @click.option('--path', '-p')
 @click.argument('token', required=False, nargs=-1)
-def remove_token(ctx, topic_hierarchy, path, token):
-    """Delete one to many tokens for a topic"""
+def remove_token(ctx, metadata_id, path, token):
+    """Delete one to many tokens for a dataset"""
+
 
-    if topic_hierarchy is not None:
-        th, _ = validate_and_load(topic_hierarchy)
-        topic = th.dotpath
+    if metadata_id is not None:
+        path = metadata_id
     elif path is not None:
-        topic = path
+        path = path
     else:
-        raise click.ClickException('Missing path or topic hierarchy')
+        raise click.ClickException('Missing path or metadata_id')
 
-    if delete_token(topic, token):
+    if delete_token(path, token):
         click.echo('Token successfully deleted')
 
 
 auth.add_command(add_token)
 auth.add_command(remove_token)
-auth.add_command(has_access_topic)
+auth.add_command(has_access_dataset)
 auth.add_command(has_access_path)
-auth.add_command(is_restricted_topic)
+auth.add_command(is_restricted_dataset)
 auth.add_command(is_restricted_path)

wis2box-management/wis2box/data/base.py

@@ -25,11 +25,12 @@
 import re
 from typing import Iterator, Union
 
-from wis2box.env import (STORAGE_INCOMING, STORAGE_PUBLIC,
+from wis2box.env import (STORAGE_PUBLIC,
                          STORAGE_SOURCE, BROKER_PUBLIC,
                          DOCKER_BROKER)
 from wis2box.storage import exists, get_data, put_data
-from wis2box.topic_hierarchy import TopicHierarchy
+from wis2box.dataset import Dataset
+
 from wis2box.plugin import load_plugin, PLUGINS
 
 from wis2box.pubsub.message import WISNotificationMessage
@@ -52,7 +53,7 @@ def __init__(self, defs: dict) -> None:
         LOGGER.debug('Parsing resource mappings')
         self.filename = None
         self.incoming_filepath = None
-        self.topic_hierarchy = TopicHierarchy(defs['topic_hierarchy'])
+        self.dataset = Dataset(defs['dataset'])
         self.template = defs.get('template', None)
         self.file_filter = defs.get('pattern', '.*')
         self.enable_notification = defs.get('notify', False)
@@ -91,10 +92,6 @@ def setup_discovery_metadata(self, discovery_metadata: dict) -> None:
         """
 
         self.discovery_metadata = discovery_metadata
-
-        self.topic_hierarchy = TopicHierarchy(
-            discovery_metadata['metadata']['identifier'])
-
         self.country = discovery_metadata['wis2box']['country']
         self.centre_id = discovery_metadata['wis2box']['centre_id']
 
@@ -146,13 +143,13 @@ def notify(self, identifier: str, storage_path: str,
         LOGGER.info('Publishing WISNotificationMessage to public broker')
         LOGGER.debug(f'Prepare message for: {storage_path}')
 
-        topic = f'origin/a/wis2/{self.topic_hierarchy.dirpath}'
-        data_id = topic.replace('origin/a/wis2/', '')
+        topic = self.dataset.topic_hierarchy
+        metadata_id = self.dataset.metadata_id
 
         operation = 'create' if is_update is False else 'update'
 
         wis_message = WISNotificationMessage(
-            identifier, data_id, storage_path, datetime_, geometry,
+            identifier, metadata_id, storage_path, datetime_, geometry,
             wigos_station_identifier, operation)
 
         # load plugin for public broker
@@ -299,17 +296,6 @@ def files(self) -> Iterator[str]:
 
                 yield f'{STORAGE_PUBLIC}/{rfp}/{identifier}.{format_}'
 
-    @property
-    def directories(self):
-        """Dataset directories"""
-
-        dirpath = self.topic_hierarchy.dirpath
-
-        return {
-            'incoming': f'{STORAGE_INCOMING}/{dirpath}',
-            'public': f'{STORAGE_PUBLIC}/{dirpath}'
-        }
-
     def get_public_filepath(self):
         """Public filepath"""
 

wis2box-management/wis2box/handler.py

@@ -25,7 +25,7 @@
 
 from wis2box.api import upsert_collection_item
 from wis2box.storage import get_data
-from wis2box.topic_hierarchy import validate_and_load
+from wis2box.dataset import validate_and_load
 
 from wis2box.plugin import load_plugin
 from wis2box.plugin import PLUGINS
@@ -37,7 +37,7 @@
 
 class Handler:
     def __init__(self, filepath: str,
-                 topic_hierarchy: str = None,
+                 metadata_id: str = None,
                  data_mappings: dict = None) -> None:
         self.filepath = filepath
         self.plugins = ()
@@ -56,8 +56,7 @@ def __init__(self, filepath: str,
         if self.filepath.startswith('http'):
             self.input_bytes = get_data(self.filepath)
 
-        if topic_hierarchy is not None:
-            th = topic_hierarchy
+        if metadata_id is not None:
             fuzzy = False
         else:
             th = self.filepath
@@ -68,10 +67,10 @@ def __init__(self, filepath: str,
             raise NotHandledError(msg)
 
         try:
-            self.topic_hierarchy, self.plugins = validate_and_load(
+            self.metadata_id, self.plugins = validate_and_load(
                 th, data_mappings, self.filetype, fuzzy=fuzzy)
         except Exception as err:
-            msg = f'Topic Hierarchy validation error: {err}'
+            msg = f'Path validation error: {err}'
             # errors in public storage are not handled
             if STORAGE_PUBLIC in self.filepath:
                 raise NotHandledError(msg)

wis2box-management/wis2box/metadata/discovery.py

@@ -178,8 +178,11 @@ def publish_broker_message(record: dict, storage_path: str,
     topic = f'origin/a/wis2/{centre_id.lower()}/metadata'  # noqa
 
     datetime_ = datetime.strptime(record['properties']['created'], '%Y-%m-%dT%H:%M:%SZ')  # noqa
-    wis_message = WISNotificationMessage(record['id'], topic, storage_path,
-                                         datetime_, record['geometry']).dumps()
+    wis_message = WISNotificationMessage(identifier=record['id'],
+                                         metadata_id=None,
+                                         filepath=storage_path,
+                                         datetime_=datetime_,
+                                         geometry=record['geometry']).dumps()
 
     # load plugin for plugin-broker
     defs = {

wis2box-management/wis2box/pubsub/message.py

@@ -53,15 +53,15 @@ class PubSubMessage:
     Generic message class
     """
 
-    def __init__(self, type_: str, identifier: str, topic: str, filepath: str,
+    def __init__(self, type_: str, identifier: str, metadata_id: str, filepath: str,
                  datetime_: datetime, geometry: dict = None,
                  wigos_station_identifier: str = None) -> None:
         """
         Initializer
 
         :param type_: message type
         :param identifier: identifier
-        :param topic: topic
+        :param metadata_id: metadata_id
         :param filepath: `Path` of file
         :param datetime_: `datetime` object of temporal aspect of data
         :param geometry: `dict` of GeoJSON geometry object
@@ -128,16 +128,16 @@ def _generate_checksum(self, bytes, algorithm: SecureHashAlgorithms) -> str:  #
 
 
 class WISNotificationMessage(PubSubMessage):
-    def __init__(self, identifier: str, topic: str, filepath: str,
+    def __init__(self, identifier: str, metadata_id: str, filepath: str,
                  datetime_: str, geometry=None, wigos_station_identifier=None,
                  operation: str = 'create') -> None:
 
         super().__init__('wis2-notification-message', identifier,
-                         topic, filepath, datetime_, geometry)
+                         metadata_id, filepath, datetime_, geometry)
 
-        data_id = f'{topic}/{self.identifier}'.replace('origin/a/wis2/', '')
+        data_id = f'{metadata_id}/{self.identifier}'.replace('origin/a/wis2/', '')
 
-        if '/metadata' in topic:
+        if '/metadata' in metadata_id:
             mimetype = 'application/geo+json'
         else:
             suffix = self.filepath.split('.')[-1]