Introduce Index meta-database

By calling `./run.sh index`, the index meta-database will be run at port
5001. This means one can run both servers on the same machine.

.gitignore

@@ -115,3 +115,4 @@ venv.bak/
 
 Untitled.ipynb
 local_openapi.json
+local_index_openapi.json

optimade/models/baseinfo.py

@@ -48,17 +48,17 @@ class BaseInfoAttributes(BaseModel):
         "(i.e., the default is for is_index to be false).",
     )
 
-    @validator("entry_types_by_format", whole=True)
-    def formats_and_endpoints_must_be_valid(cls, v, values):
-        for format_, endpoints in v.items():
-            if format_ not in values["formats"]:
-                raise ValueError(f"'{format_}' must be listed in formats to be valid")
-            for endpoint in endpoints:
-                if endpoint not in values["available_endpoints"]:
-                    raise ValueError(
-                        f"'{endpoint}' must be listed in available_endpoints to be valid"
-                    )
-        return v
+    # @validator("entry_types_by_format", whole=True, check_fields=False)
+    # def formats_and_endpoints_must_be_valid(cls, v, values):
+    #     for format_, endpoints in v.items():
+    #         if format_ not in values["formats"]:
+    #             raise ValueError(f"'{format_}' must be listed in formats to be valid")
+    #         for endpoint in endpoints:
+    #             if endpoint not in values["available_endpoints"]:
+    #                 raise ValueError(
+    #                     f"'{endpoint}' must be listed in available_endpoints to be valid"
+    #                 )
+    #     return v
 
 
 class BaseInfoResource(Resource):

optimade/models/toplevel.py

@@ -13,6 +13,7 @@
 from .utils import NonnegativeInt
 from .baseinfo import BaseInfoResource
 from .entries import EntryInfoResource, EntryResource
+from .index_metadb import IndexInfoResource
 from .links import LinksResource
 from .optimade_json import Error, Success, Failure, Warnings
 from .references import ReferenceResource
@@ -27,6 +28,7 @@
     "ResponseMeta",
     "ErrorResponse",
     "EntryInfoResponse",
+    "IndexInfoResponse",
     "InfoResponse",
     "LinksResponse",
     "EntryResponseOne",
@@ -180,6 +182,11 @@ class ErrorResponse(Failure):
     errors: List[Error] = Schema(...)
 
 
+class IndexInfoResponse(Success):
+    meta: Optional[ResponseMeta] = Schema(...)
+    data: IndexInfoResource = Schema(...)
+
+
 class EntryInfoResponse(Success):
     meta: Optional[ResponseMeta] = Schema(...)
     data: EntryInfoResource = Schema(...)

optimade/server/config.ini

@@ -8,13 +8,14 @@ STRUCTURES_COLLECTION = structures
 [IMPLEMENTATION]
 PAGE_LIMIT = 500
 VERSION = 0.10.0
+DEFAULT_DB = test_server
 
 [PROVIDER]
 prefix = _exmpl_
 name = Example provider
 description = Provider used for examples, not to be assigned to a real database
 homepage = http://example.com
-index_base_url = http://example.com/index/optimade
+index_base_url = http://localhost:5001/index/optimade
 
 [structures]
 band_gap :

optimade/server/config.py

@@ -41,6 +41,7 @@ class ServerConfig(Config):
 
     page_limit = 500
     version = "0.10.0"
+    default_db = "test_server"
 
     provider = {
         "prefix": "_exmpl_",
@@ -69,6 +70,7 @@ def load_from_ini(self):
             "IMPLEMENTATION", "PAGE_LIMIT", fallback=self.page_limit
         )
         self.version = config.get("IMPLEMENTATION", "VERSION", fallback=self.version)
+        self.default_db = config.get("IMPLEMENTATION", "DEFAULT_DB", fallback=self.default_db)
 
         if "PROVIDER" in config.sections():
             self.provider = dict(config["PROVIDER"])
@@ -110,6 +112,7 @@ def load_from_json(self):
 
         self.page_limit = int(config.get("page_limit", self.page_limit))
         self.version = config.get("version", self.version)
+        self.default_db = config.get("default_db", self.default_db)
 
         self.provider = config.get("provider", self.provider)
         self.provider_fields = set(config.get("provider_fields", self.provider_fields))

optimade/server/index_links.json

@@ -0,0 +1,12 @@
+[
+  {
+    "_id": {
+      "$oid": "746573745f73657276657263"
+    },
+    "task_id": "test_server",
+    "type": "child",
+    "name": "OPTiMaDe API",
+    "description": "The [Open Databases Integration for Materials Design (OPTiMaDe) consortium](http://http://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.",
+    "base_url": "http://localhost:5000/optimade"
+  }
+]

optimade/server/main_index.py

@@ -0,0 +1,80 @@
+import json
+from pathlib import Path
+
+from pydantic import ValidationError
+from fastapi import FastAPI
+from fastapi.exceptions import RequestValidationError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+
+from .config import CONFIG
+from .routers import index_info, links
+
+import optimade.server.exception_handlers as exc_handlers
+
+
+app = FastAPI(
+    title="OPTiMaDe API - Index meta-database",
+    description=(
+        "The [Open Databases Integration for Materials Design (OPTiMaDe) consortium]"
+        "(http://http://www.optimade.org/) aims to make materials databases interoperational "
+        "by developing a common REST API.\n"
+        'This is the "special" index meta-database.'
+    ),
+    version=CONFIG.version,
+    docs_url="/index/optimade/extensions/docs",
+    redoc_url="/index/optimade/extensions/redoc",
+    openapi_url="/index/optimade/extensions/openapi.json",
+)
+
+
+index_links_path = Path(__file__).resolve().parent.joinpath("index_links.json")
+if not CONFIG.use_real_mongo and index_links_path.exists():
+    import bson.json_util
+    from .routers.links import links_coll
+
+    print("loading index links...")
+    with open(index_links_path) as f:
+        data = json.load(f)
+        print("inserting index links into collection...")
+        links_coll.collection.insert_many(
+            bson.json_util.loads(bson.json_util.dumps(data))
+        )
+    print("done inserting index links...")
+
+
+app.add_exception_handler(StarletteHTTPException, exc_handlers.http_exception_handler)
+app.add_exception_handler(
+    RequestValidationError, exc_handlers.request_validation_exception_handler
+)
+app.add_exception_handler(ValidationError, exc_handlers.validation_exception_handler)
+app.add_exception_handler(Exception, exc_handlers.general_exception_handler)
+
+
+# Create the following prefixes:
+#   /optimade
+#   /optimade/vMajor (but only if Major >= 1)
+#   /optimade/vMajor.Minor
+#   /optimade/vMajor.Minor.Patch
+valid_prefixes = ["/index/optimade"]
+version = [int(_) for _ in app.version.split(".")]
+while version:
+    if version[0] or len(version) >= 2:
+        valid_prefixes.append(
+            "/index/optimade/v{}".format(".".join([str(_) for _ in version]))
+        )
+    version.pop(-1)
+
+for prefix in valid_prefixes:
+    app.include_router(index_info.router, prefix=prefix)
+    app.include_router(links.router, prefix=prefix)
+
+
+def update_schema(app):
+    """Update OpenAPI schema in file 'local_index_openapi.json'"""
+    with open("local_index_openapi.json", "w") as f:
+        json.dump(app.openapi(), f, indent=2)
+
+
+@app.on_event("startup")
+async def startup_event():
+    update_schema(app)

optimade/server/routers/index_info.py

@@ -0,0 +1,50 @@
+from typing import Union
+
+from fastapi import APIRouter
+from starlette.requests import Request
+
+from optimade.models import (
+    ErrorResponse,
+    IndexInfoResponse,
+    IndexInfoAttributes,
+    IndexInfoResource,
+    IndexRelationship,
+)
+
+from optimade.server.config import CONFIG
+
+from .utils import meta_values
+
+
+router = APIRouter()
+
+
+@router.get(
+    "/info",
+    response_model=Union[IndexInfoResponse, ErrorResponse],
+    response_model_skip_defaults=False,
+    tags=["Info"],
+)
+def get_info(request: Request):
+    return IndexInfoResponse(
+        meta=meta_values(str(request.url), 1, 1, more_data_available=False),
+        data=IndexInfoResource(
+            attributes=IndexInfoAttributes(
+                api_version=f"v{CONFIG.version}",
+                available_api_versions=[
+                    {
+                        "url": f"{CONFIG.provider['index_base_url']}/v{CONFIG.version}/",
+                        "version": f"{CONFIG.version}",
+                    }
+                ],
+                entry_types_by_format={"json": ["links"]},
+                available_endpoints=["info", "links"],
+                is_index=True,
+            ),
+            relationships={
+                "default": IndexRelationship(
+                    data={"type": "child", "id": CONFIG.default_db}
+                )
+            },
+        ),
+    )

optimade/server/tests/test_links.json

@@ -1,15 +1,12 @@
 [
-    {
-      "_id": {
-        "$oid": "5cfb441f053b174410700d03"
-      },
-      "id": "index",
-      "last_modified": {
-        "$date": "2019-11-12T14:24:37.331Z"
-      },
-      "type": "parent",
-      "name": "Index meta-database",
-      "description": "Index for example's OPTiMaDe databases",
-      "base_url": "http://example.com/optimade/index"
-    }
-  ]
+  {
+    "_id": {
+      "$oid": "696e646578706172656e7430"
+    },
+    "task_id": "index",
+    "type": "parent",
+    "name": "Index meta-database",
+    "description": "Index for example's OPTiMaDe databases",
+    "base_url": "http://localhost:5001/index/optimade"
+  }
+]

run.sh

@@ -1,2 +1,11 @@
 #!/bin/bash
-uvicorn optimade.server.main:app --reload --port 5000
+if [ "$1" == "index" ]
+then
+    MAIN="main_index"
+    PORT=5001
+else
+    MAIN="main"
+    PORT=5000
+fi
+
+uvicorn optimade.server.$MAIN:app --reload --port $PORT