From 26ad287d687d64666c0ea6178b84b21a94c5f2a5 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 28 Nov 2019 19:02:12 +0100 Subject: [PATCH] Introduce Index meta-database By calling `./run.sh index`, the index meta-database will be run at port 5001. This means one can run both servers on the same machine. --- .gitignore | 1 + optimade/models/baseinfo.py | 22 ++++---- optimade/models/toplevel.py | 7 +++ optimade/server/config.ini | 3 +- optimade/server/config.py | 3 + optimade/server/index_links.json | 12 ++++ optimade/server/main_index.py | 80 +++++++++++++++++++++++++++ optimade/server/routers/index_info.py | 50 +++++++++++++++++ optimade/server/tests/test_links.json | 25 ++++----- run.sh | 11 +++- 10 files changed, 187 insertions(+), 27 deletions(-) create mode 100644 optimade/server/index_links.json create mode 100644 optimade/server/main_index.py create mode 100644 optimade/server/routers/index_info.py diff --git a/.gitignore b/.gitignore index 7c3ad8fc7..7f8135d2b 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,4 @@ venv.bak/ Untitled.ipynb local_openapi.json +local_index_openapi.json diff --git a/optimade/models/baseinfo.py b/optimade/models/baseinfo.py index 920248ff2..00446752c 100644 --- a/optimade/models/baseinfo.py +++ b/optimade/models/baseinfo.py @@ -48,17 +48,17 @@ class BaseInfoAttributes(BaseModel): "(i.e., the default is for is_index to be false).", ) - @validator("entry_types_by_format", whole=True) - def formats_and_endpoints_must_be_valid(cls, v, values): - for format_, endpoints in v.items(): - if format_ not in values["formats"]: - raise ValueError(f"'{format_}' must be listed in formats to be valid") - for endpoint in endpoints: - if endpoint not in values["available_endpoints"]: - raise ValueError( - f"'{endpoint}' must be listed in available_endpoints to be valid" - ) - return v + # @validator("entry_types_by_format", whole=True, check_fields=False) + # def formats_and_endpoints_must_be_valid(cls, v, values): + # for format_, endpoints in v.items(): + # if format_ not in values["formats"]: + # raise ValueError(f"'{format_}' must be listed in formats to be valid") + # for endpoint in endpoints: + # if endpoint not in values["available_endpoints"]: + # raise ValueError( + # f"'{endpoint}' must be listed in available_endpoints to be valid" + # ) + # return v class BaseInfoResource(Resource): diff --git a/optimade/models/toplevel.py b/optimade/models/toplevel.py index e61eb9375..85fdb9456 100644 --- a/optimade/models/toplevel.py +++ b/optimade/models/toplevel.py @@ -13,6 +13,7 @@ from .utils import NonnegativeInt from .baseinfo import BaseInfoResource from .entries import EntryInfoResource, EntryResource +from .index_metadb import IndexInfoResource from .links import LinksResource from .optimade_json import Error, Success, Failure, Warnings from .references import ReferenceResource @@ -27,6 +28,7 @@ "ResponseMeta", "ErrorResponse", "EntryInfoResponse", + "IndexInfoResponse", "InfoResponse", "LinksResponse", "EntryResponseOne", @@ -180,6 +182,11 @@ class ErrorResponse(Failure): errors: List[Error] = Schema(...) +class IndexInfoResponse(Success): + meta: Optional[ResponseMeta] = Schema(...) + data: IndexInfoResource = Schema(...) + + class EntryInfoResponse(Success): meta: Optional[ResponseMeta] = Schema(...) data: EntryInfoResource = Schema(...) diff --git a/optimade/server/config.ini b/optimade/server/config.ini index aa66816e4..a0f2e7f0b 100644 --- a/optimade/server/config.ini +++ b/optimade/server/config.ini @@ -8,13 +8,14 @@ STRUCTURES_COLLECTION = structures [IMPLEMENTATION] PAGE_LIMIT = 500 VERSION = 0.10.0 +DEFAULT_DB = test_server [PROVIDER] prefix = _exmpl_ name = Example provider description = Provider used for examples, not to be assigned to a real database homepage = http://example.com -index_base_url = http://example.com/index/optimade +index_base_url = http://localhost:5001/index/optimade [structures] band_gap : diff --git a/optimade/server/config.py b/optimade/server/config.py index e0d115a4f..3d76744b2 100644 --- a/optimade/server/config.py +++ b/optimade/server/config.py @@ -41,6 +41,7 @@ class ServerConfig(Config): page_limit = 500 version = "0.10.0" + default_db = "test_server" provider = { "prefix": "_exmpl_", @@ -69,6 +70,7 @@ def load_from_ini(self): "IMPLEMENTATION", "PAGE_LIMIT", fallback=self.page_limit ) self.version = config.get("IMPLEMENTATION", "VERSION", fallback=self.version) + self.default_db = config.get("IMPLEMENTATION", "DEFAULT_DB", fallback=self.default_db) if "PROVIDER" in config.sections(): self.provider = dict(config["PROVIDER"]) @@ -110,6 +112,7 @@ def load_from_json(self): self.page_limit = int(config.get("page_limit", self.page_limit)) self.version = config.get("version", self.version) + self.default_db = config.get("default_db", self.default_db) self.provider = config.get("provider", self.provider) self.provider_fields = set(config.get("provider_fields", self.provider_fields)) diff --git a/optimade/server/index_links.json b/optimade/server/index_links.json new file mode 100644 index 000000000..f88771704 --- /dev/null +++ b/optimade/server/index_links.json @@ -0,0 +1,12 @@ +[ + { + "_id": { + "$oid": "746573745f73657276657263" + }, + "task_id": "test_server", + "type": "child", + "name": "OPTiMaDe API", + "description": "The [Open Databases Integration for Materials Design (OPTiMaDe) consortium](http://http://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.", + "base_url": "http://localhost:5000/optimade" + } +] diff --git a/optimade/server/main_index.py b/optimade/server/main_index.py new file mode 100644 index 000000000..360cc311f --- /dev/null +++ b/optimade/server/main_index.py @@ -0,0 +1,80 @@ +import json +from pathlib import Path + +from pydantic import ValidationError +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .config import CONFIG +from .routers import index_info, links + +import optimade.server.exception_handlers as exc_handlers + + +app = FastAPI( + title="OPTiMaDe API - Index meta-database", + description=( + "The [Open Databases Integration for Materials Design (OPTiMaDe) consortium]" + "(http://http://www.optimade.org/) aims to make materials databases interoperational " + "by developing a common REST API.\n" + 'This is the "special" index meta-database.' + ), + version=CONFIG.version, + docs_url="/index/optimade/extensions/docs", + redoc_url="/index/optimade/extensions/redoc", + openapi_url="/index/optimade/extensions/openapi.json", +) + + +index_links_path = Path(__file__).resolve().parent.joinpath("index_links.json") +if not CONFIG.use_real_mongo and index_links_path.exists(): + import bson.json_util + from .routers.links import links_coll + + print("loading index links...") + with open(index_links_path) as f: + data = json.load(f) + print("inserting index links into collection...") + links_coll.collection.insert_many( + bson.json_util.loads(bson.json_util.dumps(data)) + ) + print("done inserting index links...") + + +app.add_exception_handler(StarletteHTTPException, exc_handlers.http_exception_handler) +app.add_exception_handler( + RequestValidationError, exc_handlers.request_validation_exception_handler +) +app.add_exception_handler(ValidationError, exc_handlers.validation_exception_handler) +app.add_exception_handler(Exception, exc_handlers.general_exception_handler) + + +# Create the following prefixes: +# /optimade +# /optimade/vMajor (but only if Major >= 1) +# /optimade/vMajor.Minor +# /optimade/vMajor.Minor.Patch +valid_prefixes = ["/index/optimade"] +version = [int(_) for _ in app.version.split(".")] +while version: + if version[0] or len(version) >= 2: + valid_prefixes.append( + "/index/optimade/v{}".format(".".join([str(_) for _ in version])) + ) + version.pop(-1) + +for prefix in valid_prefixes: + app.include_router(index_info.router, prefix=prefix) + app.include_router(links.router, prefix=prefix) + + +def update_schema(app): + """Update OpenAPI schema in file 'local_index_openapi.json'""" + with open("local_index_openapi.json", "w") as f: + json.dump(app.openapi(), f, indent=2) + + +@app.on_event("startup") +async def startup_event(): + update_schema(app) diff --git a/optimade/server/routers/index_info.py b/optimade/server/routers/index_info.py new file mode 100644 index 000000000..74a94071a --- /dev/null +++ b/optimade/server/routers/index_info.py @@ -0,0 +1,50 @@ +from typing import Union + +from fastapi import APIRouter +from starlette.requests import Request + +from optimade.models import ( + ErrorResponse, + IndexInfoResponse, + IndexInfoAttributes, + IndexInfoResource, + IndexRelationship, +) + +from optimade.server.config import CONFIG + +from .utils import meta_values + + +router = APIRouter() + + +@router.get( + "/info", + response_model=Union[IndexInfoResponse, ErrorResponse], + response_model_skip_defaults=False, + tags=["Info"], +) +def get_info(request: Request): + return IndexInfoResponse( + meta=meta_values(str(request.url), 1, 1, more_data_available=False), + data=IndexInfoResource( + attributes=IndexInfoAttributes( + api_version=f"v{CONFIG.version}", + available_api_versions=[ + { + "url": f"{CONFIG.provider['index_base_url']}/v{CONFIG.version}/", + "version": f"{CONFIG.version}", + } + ], + entry_types_by_format={"json": ["links"]}, + available_endpoints=["info", "links"], + is_index=True, + ), + relationships={ + "default": IndexRelationship( + data={"type": "child", "id": CONFIG.default_db} + ) + }, + ), + ) diff --git a/optimade/server/tests/test_links.json b/optimade/server/tests/test_links.json index a1b4976b4..2eac40d8e 100644 --- a/optimade/server/tests/test_links.json +++ b/optimade/server/tests/test_links.json @@ -1,15 +1,12 @@ [ - { - "_id": { - "$oid": "5cfb441f053b174410700d03" - }, - "id": "index", - "last_modified": { - "$date": "2019-11-12T14:24:37.331Z" - }, - "type": "parent", - "name": "Index meta-database", - "description": "Index for example's OPTiMaDe databases", - "base_url": "http://example.com/optimade/index" - } - ] \ No newline at end of file + { + "_id": { + "$oid": "696e646578706172656e7430" + }, + "task_id": "index", + "type": "parent", + "name": "Index meta-database", + "description": "Index for example's OPTiMaDe databases", + "base_url": "http://localhost:5001/index/optimade" + } +] diff --git a/run.sh b/run.sh index 24dc70596..ed45ff7f5 100755 --- a/run.sh +++ b/run.sh @@ -1,2 +1,11 @@ #!/bin/bash -uvicorn optimade.server.main:app --reload --port 5000 +if [ "$1" == "index" ] +then + MAIN="main_index" + PORT=5001 +else + MAIN="main" + PORT=5000 +fi + +uvicorn optimade.server.$MAIN:app --reload --port $PORT