improve management of api-config (#722)

* improve management of api-config

* Delete tests/data/downloads/.session-info.json

* Update __init__.py

---------

Co-authored-by: Tom Kralidis <[email protected]>

docker-compose.yml

@@ -44,8 +44,6 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    volumes:
-      - api-config:/data/wis2box/config/pygeoapi/:rw
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost/oapi/admin/resources"]
       interval: 5s
@@ -157,5 +155,4 @@ volumes:
   es-data:
   minio-data:
   auth-data:
-  api-config:
   htpasswd:

wis2box-management/wis2box/api/__init__.py

@@ -25,6 +25,8 @@
 
 from time import sleep
 
+from owslib.ogcapi.records import Records
+
 from wis2box import cli_helpers
 from wis2box.api.backend import load_backend
 from wis2box.api.config import load_config
@@ -102,7 +104,6 @@ def setup_collection(meta: dict = {}) -> bool:
 
     backend = load_backend()
     if not backend.has_collection(data_name):
-
         if not backend.add_collection(data_name):
             msg = f'Unable to setup backend for collection {data_name}'
             LOGGER.error(msg)
@@ -241,13 +242,29 @@ def api():
 @click.pass_context
 @cli_helpers.OPTION_VERBOSITY
 def setup(ctx, verbosity):
-    """Add collection items to API backend"""
+    """Add collection items to API using discovery metadata"""
 
     api_config = load_config()
-    if not api_config.has_collection(''):
-        click.echo('API not ready')
-    else:
-        click.echo('API ready')
+    api_collections = api_config.list_collections()
+    # print resources in api_config
+    click.echo(f'Collections in api_config: {api_collections}')
+    backend = load_backend()
+    backend_collections = backend.list_collections()
+    click.echo(f'Collections in backend: {backend_collections}')
+
+    oar = Records(DOCKER_API_URL)
+    try:
+        records = oar.collection_items('discovery-metadata')
+    except Exception as err:
+        click.echo(f'Issue loading discovery-metadata: {err}')
+        return False
+    for record in records['features']:
+        metadata_id = record['id']
+        if metadata_id not in api_collections:
+            click.echo(f'Adding collection: {metadata_id}')
+            from wis2box.data import gcm
+            meta = gcm(record)
+            setup_collection(meta=meta)
 
 
 @click.command()

wis2box-management/wis2box/api/backend/base.py

@@ -40,6 +40,15 @@ def __init__(self, defs: dict) -> None:
         self.username = defs.get('username')
         self.password = defs.get('password')
 
+    def list_collections(self) -> list:
+        """
+        List collections
+
+        :returns: `list` of collection names
+        """
+
+        raise NotImplementedError()
+
     def add_collection(self, name: str) -> bool:
         """
         Add a collection

wis2box-management/wis2box/api/backend/elastic.py

@@ -196,6 +196,15 @@ def es_id(collection_id: str) -> Tuple[str]:
         """
         return collection_id.lower().replace(':', '-')
 
+    def list_collections(self) -> list:
+        """
+        List collections
+
+        :returns: `list` of collection names
+        """
+
+        return [index for index in self.conn.indices.get_alias(index="*")]
+
     def add_collection(self, collection_id: str) -> dict:
         """
         Add a collection

wis2box-management/wis2box/api/config/base.py

@@ -33,6 +33,15 @@ def __init__(self, defs: dict) -> None:
         :param defs: `dict` of connection parameters
         """
 
+    def list_collections(self) -> list:
+        """
+        List collections
+
+        :returns: `list` of collection names
+        """
+
+        raise NotImplementedError()
+
     def get_collection(self, name: str) -> dict:
         """
         Get a collection

wis2box-management/wis2box/api/config/pygeoapi.py

@@ -55,6 +55,23 @@ def __init__(self, defs: dict) -> None:
         self.http.mount('https://', adapter)
         self.http.mount('http://', adapter)
 
+    def list_collections(self) -> list:
+        """
+        List collections
+
+        :returns: `list` of collection names
+        """
+
+        collections = []
+
+        r = self.http.get(self.url)
+        r.raise_for_status()
+        resources = r.json()
+        for key in resources:
+            if resources[key]['type'] == 'collection':
+                collections.append(key)
+        return collections
+
     def get_collection(self, name: str) -> dict:
         """
         Get a collection