diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6320280dda..8068f4ba10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt - name: Run pre-commit @@ -85,6 +86,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt - name: Pass generated OpenAPI schemas through validator.swagger.io @@ -194,25 +196,29 @@ jobs: pip install -r requirements-dev.txt pip install -r requirements-http-client.txt - - name: Run all tests (using `mongomock`) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ + - name: Run non-server tests + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/ --ignore tests/server + + - name: Install latest server dependencies + run: pip install -r requirements-server.txt + + - name: Run server tests (using `mongomock`) + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'mongomock' - - name: Run server tests (using a real MongoDB) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'mongodb' - name: Run server tests (using Elasticsearch) - run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server + run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server tests/filtertransformers env: OPTIMADE_DATABASE_BACKEND: 'elastic' - name: Install adapter conversion dependencies - run: | - pip install -r requirements-client.txt + run: pip install -r requirements-client.txt - name: Setup environment for AiiDA env: @@ -305,6 +311,7 @@ jobs: pip install -U setuptools wheel pip install -e . pip install -r requirements.txt + pip install -r requirements-server.txt pip install -r requirements-dev.txt pip install -r requirements-http-client.txt pip install -r requirements-docs.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ff43cf8fe7..62c1f58264 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: name: Blacken - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-symlinks - id: check-yaml @@ -24,7 +24,7 @@ repos: args: [--markdown-linebreak-ext=md] - repo: https://github.com/pycqa/flake8 - rev: '5.0.4' + rev: '6.0.0' hooks: - id: flake8 @@ -43,3 +43,16 @@ repos: pass_filenames: false files: ^optimade/.*\.py$ description: Update the API Reference documentation whenever a Python file is touched in the code base. + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.991 + hooks: + - id: mypy + name: "MyPy" + additional_dependencies: ["types-all", "pydantic~=1.10"] + + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort + name: "isort" diff --git a/CHANGELOG.md b/CHANGELOG.md index 0969c8bd92..207a37d2f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,20 +2,60 @@ ## [Unreleased](https://github.com/Materials-Consortia/optimade-python-tools/tree/HEAD) -[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...HEAD) +[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.20.0...HEAD) -This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version. +This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages. + +Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku. + +## [v0.20.0](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) (2022-11-29) + +[Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.4...v0.20.0) + +This release continues the modularisation of the package by moving the server exceptions and warnings out into top-level modules, and removing the core dependency on FastAPI (now a server dependency only). This should allow for easier use of the `optimade.models` and `optimade.client` modules within other packages. + +Aside from that, the package now supports Python 3.11, and our example server is now deployed at [Fly.io](https://optimade.fly.dev) rather than Heroku. + +**Implemented enhancements:** + +- Add support for Python 3.11 [\#1361](https://github.com/Materials-Consortia/optimade-python-tools/issues/1361) +- Improve test diagnostics and fix deprecated Elasticsearch calls [\#1373](https://github.com/Materials-Consortia/optimade-python-tools/pull/1373) ([ml-evs](https://github.com/ml-evs)) +- Support Python 3.11 [\#1362](https://github.com/Materials-Consortia/optimade-python-tools/pull/1362) ([ml-evs](https://github.com/ml-evs)) **Fixed bugs:** -- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335) +- Elasticsearch pytest oneliner in the docs is no longer working [\#1377](https://github.com/Materials-Consortia/optimade-python-tools/issues/1377) +- Remote swagger validator has changed output format [\#1370](https://github.com/Materials-Consortia/optimade-python-tools/issues/1370) + +**Closed issues:** + +- Fully isolate server code from other submodules [\#1403](https://github.com/Materials-Consortia/optimade-python-tools/issues/1403) +- Replace https://gitlab.com/pycqa/flake8 with https://github.com/pycqa/flake8 [\#1388](https://github.com/Materials-Consortia/optimade-python-tools/issues/1388) +- OpenAPI schema should not enforce recommended constraint on `page_number` [\#1372](https://github.com/Materials-Consortia/optimade-python-tools/issues/1372) +- Pydantic models docs are broken on the mkdocs site with new renderer [\#1353](https://github.com/Materials-Consortia/optimade-python-tools/issues/1353) +- Migrate away from Heroku for demo server [\#1307](https://github.com/Materials-Consortia/optimade-python-tools/issues/1307) +- FastAPI should not be a core dependency [\#1198](https://github.com/Materials-Consortia/optimade-python-tools/issues/1198) + +**Merged pull requests:** + +- Move exceptions and warnings out of server code and separate deps [\#1405](https://github.com/Materials-Consortia/optimade-python-tools/pull/1405) ([ml-evs](https://github.com/ml-evs)) +- Complete migration from Heroku to Fly [\#1400](https://github.com/Materials-Consortia/optimade-python-tools/pull/1400) ([ml-evs](https://github.com/ml-evs)) +- Add GH actions for deploying example server to Fly [\#1396](https://github.com/Materials-Consortia/optimade-python-tools/pull/1396) ([ml-evs](https://github.com/ml-evs)) +- Support new remote swagger.io validator format [\#1371](https://github.com/Materials-Consortia/optimade-python-tools/pull/1371) ([ml-evs](https://github.com/ml-evs)) +- Do not enforce minimum value of `page_number` at model level [\#1369](https://github.com/Materials-Consortia/optimade-python-tools/pull/1369) ([ml-evs](https://github.com/ml-evs)) +- Enable `mypy` and `isort` in pre-commit & CI [\#1346](https://github.com/Materials-Consortia/optimade-python-tools/pull/1346) ([ml-evs](https://github.com/ml-evs)) +- Remove randomness from structure utils tests [\#1338](https://github.com/Materials-Consortia/optimade-python-tools/pull/1338) ([ml-evs](https://github.com/ml-evs)) +- Demote FastAPI to a server dep only [\#1199](https://github.com/Materials-Consortia/optimade-python-tools/pull/1199) ([ml-evs](https://github.com/ml-evs)) ## [v0.19.4](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) (2022-09-19) [Full Changelog](https://github.com/Materials-Consortia/optimade-python-tools/compare/v0.19.3...v0.19.4) +This is a hotfix release for #1335, a bug regarding chunked responses triggered when using the latest FastAPI version. + **Fixed bugs:** +- UnboundLocalError - `chunk_size` is not always set in middleware method [\#1335](https://github.com/Materials-Consortia/optimade-python-tools/issues/1335) - Ensure `chunk_size` is properly set when chunking responses [\#1336](https://github.com/Materials-Consortia/optimade-python-tools/pull/1336) ([ml-evs](https://github.com/ml-evs)) ## [v0.19.3](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.3) (2022-09-06) diff --git a/docs/api_reference/exceptions.md b/docs/api_reference/exceptions.md new file mode 100644 index 0000000000..da8d3b8ed3 --- /dev/null +++ b/docs/api_reference/exceptions.md @@ -0,0 +1,3 @@ +# exceptions + +::: optimade.exceptions diff --git a/docs/api_reference/utils.md b/docs/api_reference/utils.md new file mode 100644 index 0000000000..e227d8708d --- /dev/null +++ b/docs/api_reference/utils.md @@ -0,0 +1,3 @@ +# utils + +::: optimade.utils diff --git a/docs/api_reference/warnings.md b/docs/api_reference/warnings.md new file mode 100644 index 0000000000..7da06b4877 --- /dev/null +++ b/docs/api_reference/warnings.md @@ -0,0 +1,3 @@ +# warnings + +::: optimade.warnings diff --git a/docs/static/default_config.json b/docs/static/default_config.json index 627b411387..480a4713a9 100644 --- a/docs/static/default_config.json +++ b/docs/static/default_config.json @@ -13,7 +13,7 @@ "base_url": null, "implementation": { "name": "OPTIMADE Python Tools", - "version": "0.19.4", + "version": "0.20.0", "source_url": "https://github.com/Materials-Consortia/optimade-python-tools", "maintainer": {"email": "dev@optimade.org"} }, diff --git a/openapi/index_openapi.json b/openapi/index_openapi.json index f85e7085c2..6925733679 100644 --- a/openapi/index_openapi.json +++ b/openapi/index_openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.2", "info": { "title": "OPTIMADE API - Index meta-database", - "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\nThis is the \"special\" index meta-database.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) v0.19.4.", + "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\nThis is the \"special\" index meta-database.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) v0.20.0.", "version": "1.1.0" }, "paths": { diff --git a/openapi/openapi.json b/openapi/openapi.json index 4c88fd0f89..285c31372d 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.0.2", "info": { "title": "OPTIMADE API", - "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.19.4) v0.19.4.", + "description": "The [Open Databases Integration for Materials Design (OPTIMADE) consortium](https://www.optimade.org/) aims to make materials databases interoperational by developing a common REST API.\n\nThis specification is generated using [`optimade-python-tools`](https://github.com/Materials-Consortia/optimade-python-tools/tree/v0.20.0) v0.20.0.", "version": "1.1.0" }, "paths": { @@ -3359,9 +3359,9 @@ "type": "number" }, "description": "If present MUST be a list of floats expressed in a.m.u.\nElements denoting vacancies MUST have masses equal to 0.", - "x-optimade-unit": "a.m.u.", "x-optimade-support": "optional", - "x-optimade-queryable": "optional" + "x-optimade-queryable": "optional", + "x-optimade-unit": "a.m.u." }, "original_name": { "title": "Original Name", @@ -3639,9 +3639,9 @@ }, "description": "The three lattice vectors in Cartesian coordinates, in \u00e5ngstr\u00f6m (\u00c5).\n\n- **Type**: list of list of floats or unknown values.\n\n- **Requirements/Conventions**:\n - **Support**: SHOULD be supported by all implementations, i.e., SHOULD NOT be `null`.\n - **Query**: Support for queries on this property is OPTIONAL.\n If supported, filters MAY support only a subset of comparison operators.\n - MUST be a list of three vectors *a*, *b*, and *c*, where each of the vectors MUST BE a list of the vector's coordinates along the x, y, and z Cartesian coordinates.\n (Therefore, the first index runs over the three lattice vectors and the second index runs over the x, y, z Cartesian coordinates).\n - For databases that do not define an absolute Cartesian system (e.g., only defining the length and angles between vectors), the first lattice vector SHOULD be set along *x* and the second on the *xy*-plane.\n - MUST always contain three vectors of three coordinates each, independently of the elements of property `dimension_types`.\n The vectors SHOULD by convention be chosen so the determinant of the `lattice_vectors` matrix is different from zero.\n The vectors in the non-periodic directions have no significance beyond fulfilling these requirements.\n - The coordinates of the lattice vectors of non-periodic dimensions (i.e., those dimensions for which `dimension_types` is `0`) MAY be given as a list of all `null` values.\n If a lattice vector contains the value `null`, all coordinates of that lattice vector MUST be `null`.\n\n- **Examples**:\n - `[[4.0,0.0,0.0],[0.0,4.0,0.0],[0.0,1.0,4.0]]` represents a cell, where the first vector is `(4, 0, 0)`, i.e., a vector aligned along the `x` axis of length 4 \u00c5; the second vector is `(0, 4, 0)`; and the third vector is `(0, 1, 4)`.", "nullable": true, - "x-optimade-unit": "\u00c5", "x-optimade-support": "should", - "x-optimade-queryable": "optional" + "x-optimade-queryable": "optional", + "x-optimade-unit": "\u00c5" }, "cartesian_site_positions": { "title": "Cartesian Site Positions", @@ -3656,9 +3656,9 @@ }, "description": "Cartesian positions of each site in the structure.\nA site is usually used to describe positions of atoms; what atoms can be encountered at a given site is conveyed by the `species_at_sites` property, and the species themselves are described in the `species` property.\n\n- **Type**: list of list of floats\n\n- **Requirements/Conventions**:\n - **Support**: SHOULD be supported by all implementations, i.e., SHOULD NOT be `null`.\n - **Query**: Support for queries on this property is OPTIONAL.\n If supported, filters MAY support only a subset of comparison operators.\n - It MUST be a list of length equal to the number of sites in the structure, where every element is a list of the three Cartesian coordinates of a site expressed as float values in the unit angstrom (\u00c5).\n - An entry MAY have multiple sites at the same Cartesian position (for a relevant use of this, see e.g., the property `assemblies`).\n\n- **Examples**:\n - `[[0,0,0],[0,0,2]]` indicates a structure with two sites, one sitting at the origin and one along the (positive) *z*-axis, 2 \u00c5 away from the origin.", "nullable": true, - "x-optimade-unit": "\u00c5", "x-optimade-support": "should", - "x-optimade-queryable": "optional" + "x-optimade-queryable": "optional", + "x-optimade-unit": "\u00c5" }, "nsites": { "title": "Nsites", diff --git a/optimade/__init__.py b/optimade/__init__.py index 9879c3e2ed..b8d67fadc5 100644 --- a/optimade/__init__.py +++ b/optimade/__init__.py @@ -1,2 +1,2 @@ -__version__ = "0.19.4" +__version__ = "0.20.0" __api_version__ = "1.1.0" diff --git a/optimade/adapters/__init__.py b/optimade/adapters/__init__.py index f1be4dec86..6de6409432 100644 --- a/optimade/adapters/__init__.py +++ b/optimade/adapters/__init__.py @@ -3,5 +3,4 @@ from .references import * # noqa: F403 from .structures import * # noqa: F403 - -__all__ = exceptions.__all__ + references.__all__ + structures.__all__ # noqa: F405 +__all__ = exceptions.__all__ + references.__all__ + structures.__all__ # type: ignore[name-defined] # noqa: F405 diff --git a/optimade/adapters/base.py b/optimade/adapters/base.py index 27dcc3af0b..d084908760 100644 --- a/optimade/adapters/base.py +++ b/optimade/adapters/base.py @@ -19,13 +19,12 @@ and [`StructureResource`][optimade.models.structures.StructureResource]s, respectively. """ import re -from typing import Union, Dict, Callable, Any, Tuple, List, Type +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union from pydantic import BaseModel # pylint: disable=no-name-in-module -from optimade.models import EntryResource - from optimade.adapters.logger import LOGGER +from optimade.models.entries import EntryResource class EntryAdapter: @@ -49,15 +48,17 @@ def __init__(self, entry: dict) -> None: Parameters: entry (dict): A JSON OPTIMADE single resource entry. """ - self._entry = None - self._converted = {} + self._entry: Optional[EntryResource] = None + self._converted: Dict[str, Any] = {} - self.entry = entry + self.entry: EntryResource = entry # type: ignore[assignment] # Note that these return also the default values for otherwise non-provided properties. self._common_converters = { - "json": self.entry.json, # Return JSON serialized string, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeljson - "dict": self.entry.dict, # Return Python dict, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict + # Return JSON serialized string, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeljson + "json": self.entry.json, # type: ignore[attr-defined] + # Return Python dict, see https://pydantic-docs.helpmanual.io/usage/exporting_models/#modeldict + "dict": self.entry.dict, # type: ignore[attr-defined] } @property @@ -68,7 +69,7 @@ def entry(self) -> EntryResource: The entry resource. """ - return self._entry + return self._entry # type: ignore[return-value] @entry.setter def entry(self, value: dict) -> None: @@ -172,12 +173,12 @@ def __getattr__(self, name: str) -> Any: return res # Non-valid attribute - entry_resource_name = re.match( + _entry_resource_name = re.match( r"()", str(self.ENTRY_RESOURCE) ) entry_resource_name = ( - entry_resource_name.group(3) - if entry_resource_name is not None + _entry_resource_name.group(3) + if _entry_resource_name is not None else "UNKNOWN RESOURCE" ) raise AttributeError( diff --git a/optimade/adapters/references/__init__.py b/optimade/adapters/references/__init__.py index c0ec1d46e5..6b160763f8 100644 --- a/optimade/adapters/references/__init__.py +++ b/optimade/adapters/references/__init__.py @@ -1,4 +1,3 @@ from .adapter import Reference - __all__ = ("Reference",) diff --git a/optimade/adapters/references/adapter.py b/optimade/adapters/references/adapter.py index 148d5a19d2..ef03e5396a 100644 --- a/optimade/adapters/references/adapter.py +++ b/optimade/adapters/references/adapter.py @@ -1,5 +1,7 @@ -from optimade.models import ReferenceResource +from typing import Type + from optimade.adapters.base import EntryAdapter +from optimade.models import ReferenceResource class Reference(EntryAdapter): @@ -19,4 +21,4 @@ class Reference(EntryAdapter): """ - ENTRY_RESOURCE: ReferenceResource = ReferenceResource + ENTRY_RESOURCE: Type[ReferenceResource] = ReferenceResource diff --git a/optimade/adapters/structures/__init__.py b/optimade/adapters/structures/__init__.py index 920c7f19fc..b3ed77f8ce 100644 --- a/optimade/adapters/structures/__init__.py +++ b/optimade/adapters/structures/__init__.py @@ -1,4 +1,3 @@ from .adapter import Structure - __all__ = ("Structure",) diff --git a/optimade/adapters/structures/adapter.py b/optimade/adapters/structures/adapter.py index 68232f1360..5e9d878007 100644 --- a/optimade/adapters/structures/adapter.py +++ b/optimade/adapters/structures/adapter.py @@ -1,13 +1,14 @@ from typing import Callable, Dict, Type -from optimade.models import StructureResource + from optimade.adapters.base import EntryAdapter +from optimade.models import StructureResource from .aiida import get_aiida_structure_data from .ase import get_ase_atoms from .cif import get_cif -from .proteindatabank import get_pdb, get_pdbx_mmcif -from .pymatgen import get_pymatgen, from_pymatgen from .jarvis import get_jarvis_atoms +from .proteindatabank import get_pdb, get_pdbx_mmcif +from .pymatgen import from_pymatgen, get_pymatgen class Structure(EntryAdapter): diff --git a/optimade/adapters/structures/aiida.py b/optimade/adapters/structures/aiida.py index 88a4192339..f65c0babbf 100644 --- a/optimade/adapters/structures/aiida.py +++ b/optimade/adapters/structures/aiida.py @@ -7,17 +7,16 @@ This conversion function relies on the [`aiida-core`](https://github.com/aiidateam/aiida-core) package. """ -from warnings import warn from typing import List, Optional +from warnings import warn -from optimade.models import StructureResource as OptimadeStructure -from optimade.models import Species as OptimadeStructureSpecies - -from optimade.adapters.warnings import AdapterPackageNotFound, ConversionWarning from optimade.adapters.structures.utils import pad_cell, species_from_species_at_sites +from optimade.adapters.warnings import AdapterPackageNotFound, ConversionWarning +from optimade.models import Species as OptimadeStructureSpecies +from optimade.models import StructureResource as OptimadeStructure try: - from aiida.orm.nodes.data.structure import StructureData, Kind, Site + from aiida.orm.nodes.data.structure import Kind, Site, StructureData except (ImportError, ModuleNotFoundError): StructureData = type("StructureData", (), {}) AIIDA_NOT_FOUND = ( @@ -45,13 +44,13 @@ def get_aiida_structure_data(optimade_structure: OptimadeStructure) -> Structure attributes = optimade_structure.attributes # Convert null/None values to float("nan") - lattice_vectors, adjust_cell = pad_cell(attributes.lattice_vectors) + lattice_vectors, adjust_cell = pad_cell(attributes.lattice_vectors) # type: ignore[arg-type] structure = StructureData(cell=lattice_vectors) # If species not provided, infer data from species_at_sites species: Optional[List[OptimadeStructureSpecies]] = attributes.species if not species: - species = species_from_species_at_sites(attributes.species_at_sites) + species = species_from_species_at_sites(attributes.species_at_sites) # type: ignore[arg-type] # Add Kinds for kind in species: @@ -87,18 +86,18 @@ def get_aiida_structure_data(optimade_structure: OptimadeStructure) -> Structure ) # Add Sites - for index in range(attributes.nsites): + for index in range(attributes.nsites): # type: ignore[arg-type] # range() to ensure 1-to-1 between kind and site structure.append_site( Site( - kind_name=attributes.species_at_sites[index], - position=attributes.cartesian_site_positions[index], + kind_name=attributes.species_at_sites[index], # type: ignore[index] + position=attributes.cartesian_site_positions[index], # type: ignore[index] ) ) if adjust_cell: structure._adjust_default_cell( - pbc=[bool(dim.value) for dim in attributes.dimension_types] + pbc=[bool(dim.value) for dim in attributes.dimension_types] # type: ignore[union-attr] ) return structure diff --git a/optimade/adapters/structures/ase.py b/optimade/adapters/structures/ase.py index 84c3bdf252..93ea84bf7d 100644 --- a/optimade/adapters/structures/ase.py +++ b/optimade/adapters/structures/ase.py @@ -9,17 +9,17 @@ """ from typing import Dict -from optimade.models import Species as OptimadeStructureSpecies -from optimade.models import StructureResource as OptimadeStructure -from optimade.models import StructureFeatures - from optimade.adapters.exceptions import ConversionError from optimade.adapters.structures.utils import species_from_species_at_sites +from optimade.models import Species as OptimadeStructureSpecies +from optimade.models import StructureFeatures +from optimade.models import StructureResource as OptimadeStructure try: - from ase import Atoms, Atom + from ase import Atom, Atoms except (ImportError, ModuleNotFoundError): from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound Atoms = type("Atoms", (), {}) @@ -57,7 +57,7 @@ def get_ase_atoms(optimade_structure: OptimadeStructure) -> Atoms: species = attributes.species # If species is missing, infer data from species_at_sites if not species: - species = species_from_species_at_sites(attributes.species_at_sites) + species = species_from_species_at_sites(attributes.species_at_sites) # type: ignore[arg-type] optimade_species: Dict[str, OptimadeStructureSpecies] = {_.name: _ for _ in species} @@ -69,9 +69,9 @@ def get_ase_atoms(optimade_structure: OptimadeStructure) -> Atoms: ) atoms = [] - for site_number in range(attributes.nsites): - species_name = attributes.species_at_sites[site_number] - site = attributes.cartesian_site_positions[site_number] + for site_number in range(attributes.nsites): # type: ignore[arg-type] + species_name = attributes.species_at_sites[site_number] # type: ignore[index] + site = attributes.cartesian_site_positions[site_number] # type: ignore[index] current_species = optimade_species[species_name] diff --git a/optimade/adapters/structures/cif.py b/optimade/adapters/structures/cif.py index 5762e9ff4b..cbe35901c7 100644 --- a/optimade/adapters/structures/cif.py +++ b/optimade/adapters/structures/cif.py @@ -18,22 +18,22 @@ """ from typing import Dict -from optimade.models import Species as OptimadeStructureSpecies -from optimade.models import StructureResource as OptimadeStructure - from optimade.adapters.structures.utils import ( cell_to_cellpar, fractional_coordinates, valid_lattice_vector, ) +from optimade.models import Species as OptimadeStructureSpecies +from optimade.models import StructureResource as OptimadeStructure try: import numpy as np except ImportError: from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound - np = None + np = None # type: ignore[assignment] NUMPY_NOT_FOUND = "NumPy not found, cannot convert structure to CIF" @@ -55,7 +55,7 @@ def get_cif( # pylint: disable=too-many-locals,too-many-branches # NumPy is needed for calculations if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] cif = """# # Created from an OPTIMADE structure. @@ -71,9 +71,9 @@ def get_cif( # pylint: disable=too-many-locals,too-many-branches # Do this only if there's three non-zero lattice vectors # NOTE: This also negates handling of lattice_vectors with null/None values - if valid_lattice_vector(attributes.lattice_vectors): + if valid_lattice_vector(attributes.lattice_vectors): # type:ignore[arg-type] a_vector, b_vector, c_vector, alpha, beta, gamma = cell_to_cellpar( - attributes.lattice_vectors + attributes.lattice_vectors # type: ignore[arg-type] ) cif += ( @@ -96,8 +96,8 @@ def get_cif( # pylint: disable=too-many-locals,too-many-branches # we calculate the fractional coordinates if this is a 3D structure and we have all the necessary information. if not hasattr(attributes, "fractional_site_positions"): attributes.fractional_site_positions = fractional_coordinates( - cell=attributes.lattice_vectors, - cartesian_positions=attributes.cartesian_site_positions, + cell=attributes.lattice_vectors, # type:ignore[arg-type] + cartesian_positions=attributes.cartesian_site_positions, # type:ignore[arg-type] ) # NOTE: This is otherwise a bit ahead of its time, since this OPTIMADE property is part of an open PR. @@ -124,12 +124,12 @@ def get_cif( # pylint: disable=too-many-locals,too-many-branches sites = attributes.cartesian_site_positions species: Dict[str, OptimadeStructureSpecies] = { - species.name: species for species in attributes.species + species.name: species for species in attributes.species # type: ignore[union-attr] } - symbol_occurences = {} - for site_number in range(attributes.nsites): - species_name = attributes.species_at_sites[site_number] + symbol_occurences: Dict[str, int] = {} + for site_number in range(attributes.nsites): # type: ignore[arg-type] + species_name = attributes.species_at_sites[site_number] # type: ignore[index] site = sites[site_number] current_species = species[species_name] diff --git a/optimade/adapters/structures/jarvis.py b/optimade/adapters/structures/jarvis.py index 50bdce8348..ac7efd34b6 100644 --- a/optimade/adapters/structures/jarvis.py +++ b/optimade/adapters/structures/jarvis.py @@ -10,14 +10,15 @@ !!! success "Contributing author" This conversion function was contributed by Kamal Choudhary ([@knc6](https://github.com/knc6)). """ -from optimade.models import StructureResource as OptimadeStructure -from optimade.models import StructureFeatures from optimade.adapters.exceptions import ConversionError +from optimade.models import StructureFeatures +from optimade.models import StructureResource as OptimadeStructure try: from jarvis.core.atoms import Atoms except (ImportError, ModuleNotFoundError): from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound Atoms = type("Atoms", (), {}) @@ -54,7 +55,7 @@ def get_jarvis_atoms(optimade_structure: OptimadeStructure) -> Atoms: return Atoms( lattice_mat=attributes.lattice_vectors, - elements=[specie.name for specie in attributes.species], + elements=[specie.name for specie in attributes.species], # type: ignore[union-attr] coords=attributes.cartesian_site_positions, cartesian=True, ) diff --git a/optimade/adapters/structures/proteindatabank.py b/optimade/adapters/structures/proteindatabank.py index 73d076c243..f2e6994086 100644 --- a/optimade/adapters/structures/proteindatabank.py +++ b/optimade/adapters/structures/proteindatabank.py @@ -27,14 +27,12 @@ import numpy as np except ImportError: from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound - np = None + np = None # type: ignore[assignment] NUMPY_NOT_FOUND = "NumPy not found, cannot convert structure to your desired format" -from optimade.models import Species as OptimadeStructureSpecies -from optimade.models import StructureResource as OptimadeStructure - from optimade.adapters.structures.utils import ( cell_to_cellpar, cellpar_to_cell, @@ -42,7 +40,8 @@ scaled_cell, valid_lattice_vector, ) - +from optimade.models import Species as OptimadeStructureSpecies +from optimade.models import StructureResource as OptimadeStructure __all__ = ("get_pdb", "get_pdbx_mmcif") @@ -64,7 +63,7 @@ def get_pdbx_mmcif( # pylint: disable=too-many-locals """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] cif = """# # Created from an OPTIMADE structure. @@ -83,9 +82,9 @@ def get_pdbx_mmcif( # pylint: disable=too-many-locals attributes = optimade_structure.attributes # Do this only if there's three non-zero lattice vectors - if valid_lattice_vector(attributes.lattice_vectors): + if valid_lattice_vector(attributes.lattice_vectors): # type: ignore[arg-type] a_vector, b_vector, c_vector, alpha, beta, gamma = cell_to_cellpar( - attributes.lattice_vectors + attributes.lattice_vectors # type: ignore[arg-type] ) cif += ( @@ -108,8 +107,8 @@ def get_pdbx_mmcif( # pylint: disable=too-many-locals # we calculate the fractional coordinates if this is a 3D structure and we have all the necessary information. if not hasattr(attributes, "fractional_site_positions"): attributes.fractional_site_positions = fractional_coordinates( - cell=attributes.lattice_vectors, - cartesian_positions=attributes.cartesian_site_positions, + cell=attributes.lattice_vectors, # type: ignore[arg-type] + cartesian_positions=attributes.cartesian_site_positions, # type: ignore[arg-type] ) # NOTE: The following lines are perhaps needed to create a "valid" PDBx/mmCIF file. @@ -166,11 +165,11 @@ def get_pdbx_mmcif( # pylint: disable=too-many-locals sites = attributes.cartesian_site_positions species: Dict[str, OptimadeStructureSpecies] = { - species.name: species for species in attributes.species + species.name: species for species in attributes.species # type: ignore[union-attr] } - for site_number in range(attributes.nsites): - species_name = attributes.species_at_sites[site_number] + for site_number in range(attributes.nsites): # type: ignore[arg-type] + species_name = attributes.species_at_sites[site_number] # type: ignore[index] site = sites[site_number] current_species = species[species_name] @@ -212,14 +211,14 @@ def get_pdb( # pylint: disable=too-many-locals """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] pdb = "" attributes = optimade_structure.attributes rotation = None - if valid_lattice_vector(attributes.lattice_vectors): + if valid_lattice_vector(attributes.lattice_vectors): # type: ignore[arg-type] currentcell = np.asarray(attributes.lattice_vectors) cellpar = cell_to_cellpar(currentcell) exportedcell = cellpar_to_cell(cellpar) @@ -242,15 +241,16 @@ def get_pdb( # pylint: disable=too-many-locals pdb += "MODEL 1\n" species: Dict[str, OptimadeStructureSpecies] = { - species.name: species for species in attributes.species + species.name: species + for species in attributes.species # type:ignore[union-attr] } sites = np.asarray(attributes.cartesian_site_positions) if rotation is not None: sites = sites.dot(rotation) - for site_number in range(attributes.nsites): - species_name = attributes.species_at_sites[site_number] + for site_number in range(attributes.nsites): # type: ignore[arg-type] + species_name = attributes.species_at_sites[site_number] # type: ignore[index] site = sites[site_number] current_species = species[species_name] diff --git a/optimade/adapters/structures/pymatgen.py b/optimade/adapters/structures/pymatgen.py index fe28a616ef..274c40f55f 100644 --- a/optimade/adapters/structures/pymatgen.py +++ b/optimade/adapters/structures/pymatgen.py @@ -7,22 +7,22 @@ For more information on the pymatgen code see [their documentation](https://pymatgen.org). """ -from typing import Union, Dict, List, Optional +from typing import Dict, List, Optional, Union -from optimade.models import Species as OptimadeStructureSpecies -from optimade.models import StructureResource as OptimadeStructure from optimade.adapters.structures.utils import ( species_from_species_at_sites, valid_lattice_vector, ) +from optimade.models import Species as OptimadeStructureSpecies +from optimade.models import StructureResource as OptimadeStructure from optimade.models import StructureResourceAttributes - try: - from pymatgen.core import Structure, Molecule, Composition, Lattice + from pymatgen.core import Composition, Lattice, Molecule, Structure except (ImportError, ModuleNotFoundError): from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound Structure = type("Structure", (), {}) @@ -61,9 +61,9 @@ def get_pymatgen(optimade_structure: OptimadeStructure) -> Union[Structure, Mole warn(PYMATGEN_NOT_FOUND, AdapterPackageNotFound) return None - if valid_lattice_vector(optimade_structure.attributes.lattice_vectors) and ( - optimade_structure.attributes.nperiodic_dimensions > 0 - or any(optimade_structure.attributes.dimension_types) + if valid_lattice_vector(optimade_structure.attributes.lattice_vectors) and ( # type: ignore[arg-type] + optimade_structure.attributes.nperiodic_dimensions > 0 # type: ignore[operator] + or any(optimade_structure.attributes.dimension_types) # type: ignore[arg-type] ): return _get_structure(optimade_structure) @@ -78,9 +78,9 @@ def _get_structure(optimade_structure: OptimadeStructure) -> Structure: return Structure( lattice=Lattice(attributes.lattice_vectors, attributes.dimension_types), species=_pymatgen_species( - nsites=attributes.nsites, + nsites=attributes.nsites, # type: ignore[arg-type] species=attributes.species, - species_at_sites=attributes.species_at_sites, + species_at_sites=attributes.species_at_sites, # type: ignore[arg-type] ), coords=attributes.cartesian_site_positions, coords_are_cartesian=True, @@ -94,9 +94,9 @@ def _get_molecule(optimade_structure: OptimadeStructure) -> Molecule: return Molecule( species=_pymatgen_species( - nsites=attributes.nsites, - species=attributes.species, - species_at_sites=attributes.species_at_sites, + nsites=attributes.nsites, # type: ignore[arg-type] + species=attributes.species, # type: ignore[arg-type] + species_at_sites=attributes.species_at_sites, # type: ignore[arg-type] ), coords=attributes.cartesian_site_positions, ) @@ -193,6 +193,7 @@ def from_pymatgen(pmg_structure: Structure) -> StructureResourceAttributes: def _pymatgen_anonymized_formula_to_optimade(composition: Composition) -> str: """Construct an OPTIMADE `chemical_formula_anonymous` from a pymatgen `Composition`.""" import re + from optimade.models.utils import anonymous_element_generator return "".join( diff --git a/optimade/adapters/structures/utils.py b/optimade/adapters/structures/utils.py index 79c2aa63a5..0c2bf9f3a1 100644 --- a/optimade/adapters/structures/utils.py +++ b/optimade/adapters/structures/utils.py @@ -3,17 +3,19 @@ Most of these functions rely on the [NumPy](https://numpy.org/) library. """ -from typing import List, Optional, Tuple, Iterable -from optimade.models.structures import Vector3D +from typing import Iterable, List, Optional, Tuple, Type + from optimade.models.structures import Species as OptimadeStructureSpecies +from optimade.models.structures import Vector3D try: import numpy as np except ImportError: from warnings import warn + from optimade.adapters.warnings import AdapterPackageNotFound - np = None + np = None # type: ignore[assignment] NUMPY_NOT_FOUND = "NumPy not found, cannot convert structure to your desired format" @@ -47,7 +49,7 @@ def scaled_cell( """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] cell = np.asarray(cell) @@ -56,7 +58,7 @@ def scaled_cell( for i in range(3): vector = np.cross(cell[(i + 1) % 3], cell[(i + 2) % 3]) / volume scale.append(tuple(vector)) - return tuple(scale) + return tuple(scale) # type: ignore[return-value] def fractional_coordinates( @@ -80,12 +82,12 @@ def fractional_coordinates( """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] - cell = np.asarray(cell) - cartesian_positions = np.asarray(cartesian_positions) + cell_array = np.asarray(cell) + cartesian_positions_array = np.asarray(cartesian_positions) - fractional = np.linalg.solve(cell.T, cartesian_positions.T).T + fractional = np.linalg.solve(cell_array.T, cartesian_positions_array.T).T # Expecting a bulk 3D structure here, note, this may change in the future. # See `ase.atoms:Atoms.get_scaled_positions()` for ideas on how to handle lower dimensional structures. @@ -119,7 +121,7 @@ def cell_to_cellpar( """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] cell = np.asarray(cell) @@ -152,7 +154,7 @@ def unit_vector(x: Vector3D) -> Vector3D: """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] y = np.array(x, dtype="float") return y / np.linalg.norm(y) @@ -161,7 +163,7 @@ def unit_vector(x: Vector3D) -> Vector3D: def cellpar_to_cell( cellpar: List[float], ab_normal: Tuple[int, int, int] = (0, 0, 1), - a_direction: Tuple[int, int, int] = None, + a_direction: Optional[Tuple[int, int, int]] = None, ) -> List[Vector3D]: """Return a 3x3 cell matrix from `cellpar=[a,b,c,alpha,beta,gamma]`. @@ -205,7 +207,7 @@ def cellpar_to_cell( """ if globals().get("np", None) is None: warn(NUMPY_NOT_FOUND, AdapterPackageNotFound) - return None + return None # type: ignore[return-value] if a_direction is None: if np.linalg.norm(np.cross(ab_normal, (1, 0, 0))) < 1e-5: @@ -275,12 +277,12 @@ def cellpar_to_cell( def _pad_iter_of_iters( iterable: Iterable[Iterable], padding: Optional[float] = None, - outer: Optional[Iterable] = None, - inner: Optional[Iterable] = None, + outer: Optional[Type] = None, + inner: Optional[Type] = None, ) -> Tuple[Iterable[Iterable], bool]: """Turn any null/None values into a float in given iterable of iterables""" try: - padding = float(padding) + padding = float(padding) # type: ignore[arg-type] except (TypeError, ValueError): padding = float("nan") @@ -294,7 +296,7 @@ def _pad_iter_of_iters( if padded_iterable: padded_iterable_of_iterables = [] for inner_iterable in iterable: - new_inner_iterable = inner( + new_inner_iterable = inner( # type: ignore[misc] value if value is not None else padding for value in inner_iterable ) padded_iterable_of_iterables.append(new_inner_iterable) diff --git a/optimade/adapters/warnings.py b/optimade/adapters/warnings.py index ad21a6e724..c40c11bbb2 100644 --- a/optimade/adapters/warnings.py +++ b/optimade/adapters/warnings.py @@ -1,4 +1,4 @@ -from optimade.server.warnings import OptimadeWarning +from optimade.warnings import OptimadeWarning __all__ = ("AdapterPackageNotFound", "ConversionWarning") diff --git a/optimade/client/cli.py b/optimade/client/cli.py index 2215c96973..3aafe2d9aa 100644 --- a/optimade/client/cli.py +++ b/optimade/client/cli.py @@ -1,6 +1,6 @@ -import sys import json import pathlib +import sys import click import rich diff --git a/optimade/client/client.py b/optimade/client/client.py index 603d023666..64f53c333c 100644 --- a/optimade/client/client.py +++ b/optimade/client/client.py @@ -5,21 +5,19 @@ """ -from typing import Dict, List, Union, Iterable, Optional, Any, Tuple -from urllib.parse import urlparse -from collections import defaultdict import asyncio -import time -import json import functools - -from pydantic import AnyUrl +import json +import time +from collections import defaultdict +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from urllib.parse import urlparse # External deps that are only used in the client code try: import httpx - from rich.progress import TaskID from rich.panel import Panel + from rich.progress import TaskID except ImportError as exc: raise ImportError( "Could not find dependencies required for the `OptimadeClient`. " @@ -28,17 +26,17 @@ ) from exc -from optimade.utils import get_all_databases -from optimade.filterparser import LarkParser from optimade import __api_version__, __version__ from optimade.client.utils import ( OptimadeClientProgress, - TooManyRequestsException, QueryResults, RecoverableHTTPError, + TooManyRequestsException, silent_raise, ) -from optimade.server.exceptions import BadRequest +from optimade.exceptions import BadRequest +from optimade.filterparser import LarkParser +from optimade.utils import get_all_databases ENDPOINTS = ("structures", "references", "calculations", "info", "extensions") @@ -57,7 +55,7 @@ class OptimadeClient: """ - base_urls: Union[AnyUrl, Iterable[AnyUrl]] + base_urls: Union[str, Iterable[str]] """A list (or any iterable) of OPTIMADE base URLs to query.""" all_results: Dict[str, Dict[str, Dict[str, QueryResults]]] = defaultdict(dict) @@ -93,7 +91,7 @@ class OptimadeClient: def __init__( self, - base_urls: Union[None, AnyUrl, List[AnyUrl]] = None, + base_urls: Optional[Union[str, Iterable[str]]] = None, max_results_per_provider: int = 1000, headers: Optional[Dict] = None, http_timeout: int = 10, @@ -113,14 +111,15 @@ def __init__( """ - if not base_urls: - base_urls = get_all_databases() - self.max_results_per_provider = max_results_per_provider if self.max_results_per_provider in (-1, 0): self.max_results_per_provider = None - self.base_urls = base_urls + if not base_urls: + self.base_urls = get_all_databases() + else: + self.base_urls = base_urls + if isinstance(self.base_urls, str): self.base_urls = [self.base_urls] self.base_urls = list(self.base_urls) @@ -170,7 +169,7 @@ def __getattribute__(self, name): def get( self, - filter: str = None, + filter: Optional[str] = None, endpoint: Optional[str] = None, response_fields: Optional[List[str]] = None, sort: Optional[str] = None, @@ -226,7 +225,7 @@ def get( return {endpoint: {filter: {k: results[k].dict() for k in results}}} def count( - self, filter: str = None, endpoint: Optional[str] = None + self, filter: Optional[str] = None, endpoint: Optional[str] = None ) -> Dict[str, Dict[str, Dict[str, Optional[int]]]]: """Counts the number of results for the filter, requiring only 1 request per provider by making use of the `meta->data_returned` @@ -325,7 +324,7 @@ def _execute_queries( event_loop = None if self.use_async and not event_loop: - results = asyncio.run( + return asyncio.run( self._get_all_async( endpoint, filter, @@ -335,17 +334,15 @@ def _execute_queries( sort=sort, ) ) - else: - results = self._get_all( - endpoint, - filter, - page_limit=page_limit, - paginate=paginate, - response_fields=response_fields, - sort=sort, - ) - return results + return self._get_all( + endpoint, + filter, + page_limit=page_limit, + paginate=paginate, + response_fields=response_fields, + sort=sort, + ) def get_one( self, @@ -477,7 +474,8 @@ def _get_all( ] if results: return functools.reduce(lambda r1, r2: {**r1, **r2}, results) - return None + + return {} async def get_one_async( self, @@ -731,8 +729,9 @@ def _build_url( if sort: _sort = f"sort={sort}" - params = (_filter, _response_fields, _page_limit, _sort) - params = "&".join(p for p in params if p) + params = "&".join( + p for p in (_filter, _response_fields, _page_limit, _sort) if p + ) if params: url += f"?{params}" diff --git a/optimade/client/utils.py b/optimade/client/utils.py index 14fa348dc0..7c9415013b 100644 --- a/optimade/client/utils.py +++ b/optimade/client/utils.py @@ -1,17 +1,17 @@ -from dataclasses import dataclass, field, asdict -from contextlib import contextmanager -from typing import List, Dict, Set, Union import sys +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field +from typing import Dict, List, Set, Union +from rich.console import Console from rich.progress import ( + BarColumn, Progress, SpinnerColumn, - TimeElapsedColumn, TaskProgressColumn, - BarColumn, TextColumn, + TimeElapsedColumn, ) -from rich.console import Console __all__ = ( "RecoverableHTTPError", @@ -34,8 +34,8 @@ class TooManyRequestsException(RecoverableHTTPError): class QueryResults: """A container dataclass for the results from a given query.""" - data: Union[Dict, List[Dict]] = field(default_factory=list, init=False) - errors: List[Dict] = field(default_factory=list, init=False) + data: Union[Dict, List[Dict]] = field(default_factory=list, init=False) # type: ignore[assignment] + errors: List[str] = field(default_factory=list, init=False) links: Dict = field(default_factory=dict, init=False) included: List[Dict] = field(default_factory=list, init=False) meta: Dict = field(default_factory=dict, init=False) @@ -62,7 +62,7 @@ def update(self, page_results: Dict) -> None: # Otherwise, as is the case for `info` endpoints, `data` is a dictionary (or null) # and should be added as the only `data` field for these results. if isinstance(page_results["data"], list): - self.data.extend(page_results["data"]) + self.data.extend(page_results["data"]) # type: ignore[union-attr] elif not self.data: self.data = page_results["data"] else: diff --git a/optimade/exceptions.py b/optimade/exceptions.py new file mode 100644 index 0000000000..2a75cfd5bb --- /dev/null +++ b/optimade/exceptions.py @@ -0,0 +1,115 @@ +from abc import ABC +from typing import Any, Dict, Optional, Tuple, Type + +__all__ = ( + "OptimadeHTTPException", + "BadRequest", + "VersionNotSupported", + "Forbidden", + "NotFound", + "UnprocessableEntity", + "InternalServerError", + "NotImplementedResponse", + "POSSIBLE_ERRORS", +) + + +class OptimadeHTTPException(Exception, ABC): + """This abstract class can be subclassed to define + HTTP responses with the desired status codes, and + detailed error strings to represent in the JSON:API + error response. + + This class closely follows the `starlette.HTTPException` without + requiring it as a dependency, so that such errors can also be + raised from within client code. + + Attributes: + status_code: The HTTP status code accompanying this exception. + title: A descriptive title for this exception. + detail: An optional string containing the details of the error. + + """ + + status_code: int + title: str + detail: Optional[str] = None + headers: Optional[Dict[str, Any]] = None + + def __init__( + self, detail: Optional[str] = None, headers: Optional[dict] = None + ) -> None: + if self.status_code is None: + raise AttributeError( + "HTTPException class {self.__class__.__name__} is missing required `status_code` attribute." + ) + self.detail = detail + self.headers = headers + + def __str__(self) -> str: + return self.detail if self.detail is not None else self.__repr__() + + def __repr__(self) -> str: + class_name = self.__class__.__name__ + return f"{class_name}(status_code={self.status_code!r}, detail={self.detail!r})" + + +class BadRequest(OptimadeHTTPException): + """400 Bad Request""" + + status_code: int = 400 + title: str = "Bad Request" + + +class VersionNotSupported(OptimadeHTTPException): + """553 Version Not Supported""" + + status_code: int = 553 + title: str = "Version Not Supported" + + +class Forbidden(OptimadeHTTPException): + """403 Forbidden""" + + status_code: int = 403 + title: str = "Forbidden" + + +class NotFound(OptimadeHTTPException): + """404 Not Found""" + + status_code: int = 404 + title: str = "Not Found" + + +class UnprocessableEntity(OptimadeHTTPException): + """422 Unprocessable Entity""" + + status_code: int = 422 + title: str = "Unprocessable Entity" + + +class InternalServerError(OptimadeHTTPException): + """500 Internal Server Error""" + + status_code: int = 500 + title: str = "Internal Server Error" + + +class NotImplementedResponse(OptimadeHTTPException): + """501 Not Implemented""" + + status_code: int = 501 + title: str = "Not Implemented" + + +"""A tuple of the possible errors that can be returned by an OPTIMADE API.""" +POSSIBLE_ERRORS: Tuple[Type[OptimadeHTTPException], ...] = ( + BadRequest, + Forbidden, + NotFound, + UnprocessableEntity, + InternalServerError, + NotImplementedResponse, + VersionNotSupported, +) diff --git a/optimade/filterparser/lark_parser.py b/optimade/filterparser/lark_parser.py index 0ee167db55..01d5044cd1 100644 --- a/optimade/filterparser/lark_parser.py +++ b/optimade/filterparser/lark_parser.py @@ -5,12 +5,11 @@ """ from pathlib import Path -from typing import Dict, Tuple -from collections import defaultdict +from typing import Dict, Optional, Tuple from lark import Lark, Tree -from optimade.server.exceptions import BadRequest +from optimade.exceptions import BadRequest __all__ = ("ParserError", "LarkParser") @@ -21,7 +20,7 @@ class ParserError(Exception): """ -def get_versions() -> Dict[Tuple[int, int, int], Dict[str, str]]: +def get_versions() -> Dict[Tuple[int, int, int], Dict[str, Path]]: """Find grammar files within this package's grammar directory, returning a dictionary broken down by scraped grammar version (major, minor, patch) and variant (a string tag). @@ -30,13 +29,15 @@ def get_versions() -> Dict[Tuple[int, int, int], Dict[str, str]]: A mapping from version, variant to grammar file name. """ - dct = defaultdict(dict) + dct: Dict[Tuple[int, int, int], Dict[str, Path]] = {} for filename in Path(__file__).parent.joinpath("../grammar").glob("*.lark"): tags = filename.stem.lstrip("v").split(".") - version = tuple(map(int, tags[:3])) - variant = "default" if len(tags) == 3 else tags[-1] + version: Tuple[int, int, int] = (int(tags[0]), int(tags[1]), int(tags[2])) + variant: str = "default" if len(tags) == 3 else str(tags[-1]) + if version not in dct: + dct[version] = {} dct[version][variant] = filename - return dict(dct) + return dct AVAILABLE_PARSERS = get_versions() @@ -48,7 +49,9 @@ class LarkParser: """ - def __init__(self, version: Tuple[int, int, int] = None, variant: str = "default"): + def __init__( + self, version: Optional[Tuple[int, int, int]] = None, variant: str = "default" + ): """For a given version and variant, try to load the corresponding grammar. Parameters: @@ -78,8 +81,8 @@ def __init__(self, version: Tuple[int, int, int] = None, variant: str = "default with open(AVAILABLE_PARSERS[version][variant]) as f: self.lark = Lark(f, maybe_placeholders=False) - self.tree = None - self.filter = None + self.tree: Optional[Tree] = None + self.filter: Optional[str] = None def parse(self, filter_: str) -> Tree: """Parse a filter string into a `lark.Tree`. diff --git a/optimade/filtertransformers/base_transformer.py b/optimade/filtertransformers/base_transformer.py index d49347a067..2bb89086c5 100644 --- a/optimade/filtertransformers/base_transformer.py +++ b/optimade/filtertransformers/base_transformer.py @@ -6,15 +6,14 @@ """ import abc -from typing import Dict, Any, Type, Optional import warnings +from typing import Any, Dict, Optional, Type -from lark import Transformer, v_args, Tree +from lark import Transformer, Tree, v_args +from optimade.exceptions import BadRequest from optimade.server.mappers import BaseResourceMapper -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import UnknownProviderProperty - +from optimade.warnings import UnknownProviderProperty __all__ = ( "BaseTransformer", @@ -47,8 +46,8 @@ class Quantity: def __init__( self, name: str, - backend_field: str = None, - length_quantity: "Quantity" = None, + backend_field: Optional[str] = None, + length_quantity: Optional["Quantity"] = None, ): """Initialise the `quantity` from it's name and aliases. @@ -80,8 +79,8 @@ class BaseTransformer(Transformer, abc.ABC): """ - mapper: Optional[BaseResourceMapper] = None - operator_map: Dict[str, str] = { + mapper: Optional[Type[BaseResourceMapper]] = None + operator_map: Dict[str, Optional[str]] = { "<": None, "<=": None, ">": None, @@ -105,7 +104,7 @@ class BaseTransformer(Transformer, abc.ABC): _quantities = None def __init__( - self, mapper: BaseResourceMapper = None + self, mapper: Optional[Type[BaseResourceMapper]] = None ): # pylint: disable=super-init-not-called """Initialise the transformer object, optionally loading in a resource mapper for use when post-processing. @@ -119,7 +118,7 @@ def backend_mapping(self) -> Dict[str, Quantity]: [`Quantity`][optimade.filtertransformers.base_transformer.Quantity] object. """ return { - quantity.backend_field: quantity for _, quantity in self.quantities.items() + quantity.backend_field: quantity for _, quantity in self.quantities.items() # type: ignore[misc] } @property diff --git a/optimade/filtertransformers/elasticsearch.py b/optimade/filtertransformers/elasticsearch.py index bc9b795554..c1490a8bcd 100644 --- a/optimade/filtertransformers/elasticsearch.py +++ b/optimade/filtertransformers/elasticsearch.py @@ -1,6 +1,7 @@ -from typing import Dict, Union, Type, Optional +from typing import Dict, Optional, Type, Union + +from elasticsearch_dsl import Field, Integer, Keyword, Q, Text from lark import v_args -from elasticsearch_dsl import Q, Text, Keyword, Integer, Field from optimade.filtertransformers import BaseTransformer, Quantity from optimade.server.mappers import BaseResourceMapper @@ -43,11 +44,11 @@ class ElasticsearchQuantity(Quantity): def __init__( self, name: str, - backend_field: str = None, - length_quantity: "ElasticsearchQuantity" = None, - elastic_mapping_type: Field = None, - has_only_quantity: "ElasticsearchQuantity" = None, - nested_quantity: "ElasticsearchQuantity" = None, + backend_field: Optional[str] = None, + length_quantity: Optional["ElasticsearchQuantity"] = None, + elastic_mapping_type: Optional[Field] = None, + has_only_quantity: Optional["ElasticsearchQuantity"] = None, + nested_quantity: Optional["ElasticsearchQuantity"] = None, ): """Initialise the quantity from its name, aliases and mapping type. @@ -99,14 +100,18 @@ class ElasticTransformer(BaseTransformer): _quantity_type: Type[ElasticsearchQuantity] = ElasticsearchQuantity def __init__( - self, mapper: BaseResourceMapper = None, quantities: Dict[str, Quantity] = None + self, + mapper: Type[BaseResourceMapper], + quantities: Optional[Dict[str, Quantity]] = None, ): if quantities is not None: self.quantities = quantities super().__init__(mapper=mapper) - def _field(self, quantity: Union[str, Quantity], nested: Quantity = None) -> str: + def _field( + self, quantity: Union[str, Quantity], nested: Optional[Quantity] = None + ) -> str: """Used to unwrap from `property` to the string backend field name. If passed a `Quantity` (or a derived `ElasticsearchQuantity`), this method @@ -126,7 +131,7 @@ def _field(self, quantity: Union[str, Quantity], nested: Quantity = None) -> str """ if isinstance(quantity, str): - if quantity in self.mapper.RELATIONSHIP_ENTRY_TYPES: + if quantity in self.mapper.RELATIONSHIP_ENTRY_TYPES: # type: ignore[union-attr] raise NotImplementedError( f"Unable to filter on relationships with type {quantity!r}" ) @@ -138,16 +143,16 @@ def _field(self, quantity: Union[str, Quantity], nested: Quantity = None) -> str return quantity if nested is not None: - return "%s.%s" % (nested.backend_field, quantity.name) + return "%s.%s" % (nested.backend_field, quantity.name) # type: ignore[union-attr] - return quantity.backend_field + return quantity.backend_field # type: ignore[union-attr, return-value] def _query_op( self, quantity: Union[ElasticsearchQuantity, str], op: str, value: Union[str, float, int], - nested: ElasticsearchQuantity = None, + nested: Optional[ElasticsearchQuantity] = None, ) -> Q: """Return a range, match, or term query for the given quantity, comparison operator, and value. diff --git a/optimade/filtertransformers/mongo.py b/optimade/filtertransformers/mongo.py index 972b24a07f..862e21f5ee 100755 --- a/optimade/filtertransformers/mongo.py +++ b/optimade/filtertransformers/mongo.py @@ -5,13 +5,15 @@ import copy -import warnings import itertools -from typing import Dict, List, Any, Union -from lark import v_args, Token +import warnings +from typing import Any, Dict, List, Union + +from lark import Token, v_args + +from optimade.exceptions import BadRequest from optimade.filtertransformers.base_transformer import BaseTransformer, Quantity -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import TimestampNotRFCCompliant +from optimade.warnings import TimestampNotRFCCompliant __all__ = ("MongoTransformer",) @@ -311,7 +313,7 @@ def handle_not_or(arg: Dict[str, List]) -> Dict[str, List]: if operator in self.inverse_operator_map: filter_ = {prop: {self.inverse_operator_map[operator]: value}} if operator in ("$in", "$eq"): - filter_ = {"$and": [filter_, {prop: {"$ne": None}}]} + filter_ = {"$and": [filter_, {prop: {"$ne": None}}]} # type: ignore[dict-item] return filter_ filter_ = {prop: {"$not": expr}} diff --git a/optimade/models/__init__.py b/optimade/models/__init__.py index d5e78a1878..0185604121 100644 --- a/optimade/models/__init__.py +++ b/optimade/models/__init__.py @@ -1,24 +1,24 @@ # pylint: disable=undefined-variable -from .jsonapi import * # noqa: F403 -from .utils import * # noqa: F403 from .baseinfo import * # noqa: F403 from .entries import * # noqa: F403 from .index_metadb import * # noqa: F403 +from .jsonapi import * # noqa: F403 from .links import * # noqa: F403 from .optimade_json import * # noqa: F403 from .references import * # noqa: F403 from .responses import * # noqa: F403 from .structures import * # noqa: F403 +from .utils import * # noqa: F403 __all__ = ( - jsonapi.__all__ # noqa: F405 - + utils.__all__ # noqa: F405 - + baseinfo.__all__ # noqa: F405 - + entries.__all__ # noqa: F405 - + index_metadb.__all__ # noqa: F405 - + links.__all__ # noqa: F405 - + optimade_json.__all__ # noqa: F405 - + references.__all__ # noqa: F405 - + responses.__all__ # noqa: F405 - + structures.__all__ # noqa: F405 + jsonapi.__all__ # type: ignore[name-defined] # noqa: F405 + + utils.__all__ # type: ignore[name-defined] # noqa: F405 + + baseinfo.__all__ # type: ignore[name-defined] # noqa: F405 + + entries.__all__ # type: ignore[name-defined] # noqa: F405 + + index_metadb.__all__ # type: ignore[name-defined] # noqa: F405 + + links.__all__ # type: ignore[name-defined] # noqa: F405 + + optimade_json.__all__ # type: ignore[name-defined] # noqa: F405 + + references.__all__ # type: ignore[name-defined] # noqa: F405 + + responses.__all__ # type: ignore[name-defined] # noqa: F405 + + structures.__all__ # type: ignore[name-defined] # noqa: F405 ) diff --git a/optimade/models/baseinfo.py b/optimade/models/baseinfo.py index a7866bb566..3374b6a024 100644 --- a/optimade/models/baseinfo.py +++ b/optimade/models/baseinfo.py @@ -1,13 +1,12 @@ # pylint: disable=no-self-argument,no-name-in-module import re - from typing import Dict, List, Optional -from pydantic import BaseModel, AnyHttpUrl, Field, validator, root_validator + +from pydantic import AnyHttpUrl, BaseModel, Field, root_validator, validator from optimade.models.jsonapi import Resource from optimade.models.utils import SemanticVersion, StrictField - __all__ = ("AvailableApiVersion", "BaseInfoAttributes", "BaseInfoResource") diff --git a/optimade/models/entries.py b/optimade/models/entries.py index e642bfc4c7..6a0fc6867e 100644 --- a/optimade/models/entries.py +++ b/optimade/models/entries.py @@ -1,12 +1,12 @@ # pylint: disable=line-too-long,no-self-argument from datetime import datetime -from typing import Optional, Dict, List -from pydantic import BaseModel, validator # pylint: disable=no-name-in-module +from typing import Dict, List, Optional -from optimade.models.jsonapi import Relationships, Attributes, Resource -from optimade.models.optimade_json import Relationship, DataType -from optimade.models.utils import StrictField, OptimadeField, SupportLevel +from pydantic import BaseModel, validator # pylint: disable=no-name-in-module +from optimade.models.jsonapi import Attributes, Relationships, Resource +from optimade.models.optimade_json import DataType, Relationship +from optimade.models.utils import OptimadeField, StrictField, SupportLevel __all__ = ( "EntryRelationships", diff --git a/optimade/models/index_metadb.py b/optimade/models/index_metadb.py index 8fc82fb68e..7c48d666c2 100644 --- a/optimade/models/index_metadb.py +++ b/optimade/models/index_metadb.py @@ -1,14 +1,13 @@ # pylint: disable=no-self-argument from enum import Enum +from typing import Dict, Union -from pydantic import Field, BaseModel # pylint: disable=no-name-in-module -from typing import Union, Dict +from pydantic import BaseModel, Field # pylint: disable=no-name-in-module -from optimade.models.jsonapi import BaseResource from optimade.models.baseinfo import BaseInfoAttributes, BaseInfoResource +from optimade.models.jsonapi import BaseResource from optimade.models.utils import StrictField - __all__ = ( "IndexInfoAttributes", "RelatedLinksResource", @@ -54,7 +53,7 @@ class IndexInfoResource(BaseInfoResource): attributes: IndexInfoAttributes = Field(...) relationships: Union[ None, Dict[DefaultRelationship, IndexRelationship] - ] = StrictField( + ] = StrictField( # type: ignore[assignment] ..., title="Relationships", description="""Reference to the Links identifier object under the `links` endpoint that the provider has chosen as their 'default' OPTIMADE API database. diff --git a/optimade/models/jsonapi.py b/optimade/models/jsonapi.py index f6724e01b8..23e0db2411 100644 --- a/optimade/models/jsonapi.py +++ b/optimade/models/jsonapi.py @@ -1,15 +1,16 @@ """This module should reproduce JSON API v1.0 https://jsonapi.org/format/1.0/""" # pylint: disable=no-self-argument -from typing import Optional, Union, List, Dict, Any, Type from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Type, Union + from pydantic import ( # pylint: disable=no-name-in-module - BaseModel, AnyUrl, + BaseModel, parse_obj_as, root_validator, ) -from optimade.models.utils import StrictField +from optimade.models.utils import StrictField __all__ = ( "Meta", diff --git a/optimade/models/links.py b/optimade/models/links.py index 609fcdf792..2cf0cb3b13 100644 --- a/optimade/models/links.py +++ b/optimade/models/links.py @@ -1,17 +1,13 @@ # pylint: disable=no-self-argument from enum import Enum +from typing import Optional, Union -from pydantic import ( # pylint: disable=no-name-in-module - AnyUrl, - root_validator, -) -from typing import Union, Optional +from pydantic import AnyUrl, root_validator # pylint: disable=no-name-in-module -from optimade.models.jsonapi import Link, Attributes from optimade.models.entries import EntryResource +from optimade.models.jsonapi import Attributes, Link from optimade.models.utils import StrictField - __all__ = ( "LinksResourceAttributes", "LinksResource", diff --git a/optimade/models/optimade_json.py b/optimade/models/optimade_json.py index 25a6d3a7cf..bad7380571 100644 --- a/optimade/models/optimade_json.py +++ b/optimade/models/optimade_json.py @@ -1,16 +1,14 @@ """Modified JSON API v1.0 for OPTIMADE API""" # pylint: disable=no-self-argument,no-name-in-module -from enum import Enum - -from typing import Optional, Union, List, Dict, Type, Any from datetime import datetime +from enum import Enum +from typing import Any, Dict, List, Optional, Type, Union -from pydantic import root_validator, BaseModel, AnyHttpUrl, AnyUrl, EmailStr +from pydantic import AnyHttpUrl, AnyUrl, BaseModel, EmailStr, root_validator from optimade.models import jsonapi from optimade.models.utils import SemanticVersion, StrictField - __all__ = ( "DataType", "ResponseMetaQuery", diff --git a/optimade/models/references.py b/optimade/models/references.py index 2c9479912a..0bf4bda908 100644 --- a/optimade/models/references.py +++ b/optimade/models/references.py @@ -1,14 +1,11 @@ # pylint: disable=line-too-long,no-self-argument -from pydantic import ( # pylint: disable=no-name-in-module - BaseModel, - AnyUrl, -) from typing import List, Optional +from pydantic import AnyUrl, BaseModel + from optimade.models.entries import EntryResource, EntryResourceAttributes from optimade.models.utils import OptimadeField, SupportLevel - __all__ = ("Person", "ReferenceResourceAttributes", "ReferenceResource") diff --git a/optimade/models/responses.py b/optimade/models/responses.py index c9959650a7..01845dc82a 100644 --- a/optimade/models/responses.py +++ b/optimade/models/responses.py @@ -1,19 +1,18 @@ # pylint: disable=no-self-argument -from typing import Union, List, Optional, Dict, Any +from typing import Any, Dict, List, Optional, Union from pydantic import Field, root_validator -from optimade.models.jsonapi import Response from optimade.models.baseinfo import BaseInfoResource from optimade.models.entries import EntryInfoResource, EntryResource from optimade.models.index_metadb import IndexInfoResource +from optimade.models.jsonapi import Response from optimade.models.links import LinksResource -from optimade.models.optimade_json import Success, ResponseMeta, OptimadeError +from optimade.models.optimade_json import OptimadeError, ResponseMeta, Success from optimade.models.references import ReferenceResource from optimade.models.structures import StructureResource from optimade.models.utils import StrictField - __all__ = ( "ErrorResponse", "EntryInfoResponse", @@ -67,17 +66,17 @@ class InfoResponse(Success): class EntryResponseOne(Success): - data: Union[EntryResource, Dict[str, Any], None] = Field(...) - included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Field( + data: Union[EntryResource, Dict[str, Any], None] = Field(...) # type: ignore[assignment] + included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Field( # type: ignore[assignment] None, uniqueItems=True ) class EntryResponseMany(Success): - data: Union[List[EntryResource], List[Dict[str, Any]]] = Field( + data: Union[List[EntryResource], List[Dict[str, Any]]] = Field( # type: ignore[assignment] ..., uniqueItems=True ) - included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Field( + included: Optional[Union[List[EntryResource], List[Dict[str, Any]]]] = Field( # type: ignore[assignment] None, uniqueItems=True ) diff --git a/optimade/models/structures.py b/optimade/models/structures.py index ed0f2d9362..c9424254bc 100644 --- a/optimade/models/structures.py +++ b/optimade/models/structures.py @@ -1,25 +1,25 @@ # pylint: disable=no-self-argument,line-too-long,no-name-in-module +import math import re -import warnings import sys -import math +import warnings +from enum import Enum, IntEnum from functools import reduce -from enum import IntEnum, Enum from typing import List, Optional, Union -from pydantic import BaseModel, validator, root_validator, conlist +from pydantic import BaseModel, conlist, root_validator, validator -from optimade.models.entries import EntryResourceAttributes, EntryResource +from optimade.models.entries import EntryResource, EntryResourceAttributes from optimade.models.utils import ( + ANONYMOUS_ELEMENTS, + CHEMICAL_FORMULA_REGEXP, CHEMICAL_SYMBOLS, EXTRA_SYMBOLS, OptimadeField, StrictField, SupportLevel, - ANONYMOUS_ELEMENTS, - CHEMICAL_FORMULA_REGEXP, ) -from optimade.server.warnings import MissingExpectedField +from optimade.warnings import MissingExpectedField EXTENDED_CHEMICAL_SYMBOLS = set(CHEMICAL_SYMBOLS + EXTRA_SYMBOLS) @@ -449,7 +449,7 @@ class StructureResourceAttributes(EntryResourceAttributes): regex=CHEMICAL_FORMULA_REGEXP, ) - dimension_types: Optional[ + dimension_types: Optional[ # type: ignore[valid-type] conlist(Periodicity, min_items=3, max_items=3) ] = OptimadeField( None, @@ -497,7 +497,7 @@ class StructureResourceAttributes(EntryResourceAttributes): queryable=SupportLevel.MUST, ) - lattice_vectors: Optional[ + lattice_vectors: Optional[ # type: ignore[valid-type] conlist(Vector3D_unknown, min_items=3, max_items=3) ] = OptimadeField( None, @@ -525,7 +525,7 @@ class StructureResourceAttributes(EntryResourceAttributes): queryable=SupportLevel.OPTIONAL, ) - cartesian_site_positions: Optional[List[Vector3D]] = OptimadeField( + cartesian_site_positions: Optional[List[Vector3D]] = OptimadeField( # type: ignore[valid-type] ..., description="""Cartesian positions of each site in the structure. A site is usually used to describe positions of atoms; what atoms can be encountered at a given site is conveyed by the `species_at_sites` property, and the species themselves are described in the `species` property. diff --git a/optimade/models/utils.py b/optimade/models/utils.py index 6ba7d87a9a..603ef08ad7 100644 --- a/optimade/models/utils.py +++ b/optimade/models/utils.py @@ -1,9 +1,9 @@ import inspect -import warnings -import re import itertools +import re +import warnings from enum import Enum -from typing import Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from pydantic import Field from pydantic.fields import FieldInfo @@ -60,7 +60,7 @@ def StrictPydanticField(*args, **kwargs): def StrictField( *args: "Any", - description: str = None, + description: Optional[str] = None, **kwargs: "Any", ) -> StrictFieldInfo: """A wrapper around `pydantic.Field` that does the following: diff --git a/optimade/server/config.py b/optimade/server/config.py index 905c0ec562..f763c336df 100644 --- a/optimade/server/config.py +++ b/optimade/server/config.py @@ -1,8 +1,8 @@ # pylint: disable=no-self-argument +import warnings from enum import Enum from pathlib import Path -import warnings -from typing import Any, Dict, List, Optional, Tuple, Union, Literal +from typing import Any, Dict, List, Literal, Optional, Tuple, Union from pydantic import ( # pylint: disable=no-name-in-module AnyHttpUrl, @@ -13,10 +13,9 @@ ) from pydantic.env_settings import SettingsSourceCallable -from optimade import __version__, __api_version__ +from optimade import __api_version__, __version__ from optimade.models import Implementation, Provider - DEFAULT_CONFIG_FILE_PATH: str = str(Path.home().joinpath(".optimade.json")) """Default configuration file path. @@ -87,6 +86,7 @@ def config_file_settings(settings: BaseSettings) -> Dict[str, Any]: """ import json import os + import yaml encoding = settings.__config__.env_file_encoding diff --git a/optimade/server/data/__init__.py b/optimade/server/data/__init__.py index f8081c1e23..87060d387e 100644 --- a/optimade/server/data/__init__.py +++ b/optimade/server/data/__init__.py @@ -1,7 +1,7 @@ """ Test Data to be used with the OPTIMADE server """ -import bson.json_util from pathlib import Path +import bson.json_util data_paths = { "structures": "test_structures.json", diff --git a/optimade/server/entry_collections/elasticsearch.py b/optimade/server/entry_collections/elasticsearch.py index e0f8adef6c..a8612d6e62 100644 --- a/optimade/server/entry_collections/elasticsearch.py +++ b/optimade/server/entry_collections/elasticsearch.py @@ -1,14 +1,13 @@ import json from pathlib import Path -from typing import Tuple, List, Optional, Dict, Any, Iterable, Union +from typing import Any, Dict, Iterable, List, Optional, Tuple, Type from optimade.filtertransformers.elasticsearch import ElasticTransformer +from optimade.models import EntryResource from optimade.server.config import CONFIG +from optimade.server.entry_collections import EntryCollection from optimade.server.logger import LOGGER -from optimade.models import EntryResource from optimade.server.mappers import BaseResourceMapper -from optimade.server.entry_collections import EntryCollection - if CONFIG.database_backend.value == "elastic": from elasticsearch import Elasticsearch @@ -23,8 +22,8 @@ class ElasticCollection(EntryCollection): def __init__( self, name: str, - resource_cls: EntryResource, - resource_mapper: BaseResourceMapper, + resource_cls: Type[EntryResource], + resource_mapper: Type[BaseResourceMapper], client: Optional["Elasticsearch"] = None, ): """Initialize the ElasticCollection for the given parameters. @@ -85,7 +84,7 @@ def predefined_index(self) -> Dict[str, Any]: @staticmethod def create_elastic_index_from_mapper( - resource_mapper: BaseResourceMapper, fields: Iterable[str] + resource_mapper: Type[BaseResourceMapper], fields: Iterable[str] ) -> Dict[str, Any]: """Create a fallback elastic index based on a resource mapper. @@ -148,7 +147,7 @@ def get_id(item): def _run_db_query( self, criteria: Dict[str, Any], single_entry=False - ) -> Tuple[Union[List[Dict[str, Any]], Dict[str, Any]], int, bool]: + ) -> Tuple[List[Dict[str, Any]], int, bool]: """Run the query on the backend and collect the results. Arguments: diff --git a/optimade/server/entry_collections/entry_collections.py b/optimade/server/entry_collections/entry_collections.py index 0974e3358e..7b7bbd7e84 100644 --- a/optimade/server/entry_collections/entry_collections.py +++ b/optimade/server/entry_collections/entry_collections.py @@ -1,26 +1,29 @@ -from abc import abstractmethod, ABC -from typing import Tuple, List, Union, Dict, Any, Set -import warnings import re +import warnings +from abc import ABC, abstractmethod +from functools import lru_cache +from typing import Any, Dict, Iterable, List, Set, Tuple, Type, Union from lark import Transformer -from functools import lru_cache + +from optimade.exceptions import BadRequest, Forbidden, NotFound from optimade.filterparser import LarkParser -from optimade.models import EntryResource +from optimade.models.entries import EntryResource from optimade.server.config import CONFIG, SupportedBackend -from optimade.server.exceptions import BadRequest, Forbidden, NotFound from optimade.server.mappers import BaseResourceMapper from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams -from optimade.server.warnings import ( +from optimade.utils import set_field_to_none_if_missing_in_dict +from optimade.warnings import ( FieldValueNotRecognized, - UnknownProviderProperty, QueryParamNotUsed, + UnknownProviderProperty, ) -from optimade.utils import set_field_to_none_if_missing_in_dict def create_collection( - name: str, resource_cls: EntryResource, resource_mapper: BaseResourceMapper + name: str, + resource_cls: Type[EntryResource], + resource_mapper: Type[BaseResourceMapper], ) -> "EntryCollection": """Create an `EntryCollection` of the configured type, depending on the value of `CONFIG.database_backend`. @@ -66,8 +69,8 @@ class EntryCollection(ABC): def __init__( self, - resource_cls: EntryResource, - resource_mapper: BaseResourceMapper, + resource_cls: Type[EntryResource], + resource_mapper: Type[BaseResourceMapper], transformer: Transformer, ): """Initialize the collection for the given parameters. @@ -93,7 +96,7 @@ def __init__( for field in CONFIG.provider_fields.get(resource_mapper.ENDPOINT, []) ] - self._all_fields: Set[str] = None + self._all_fields: Set[str] = set() @abstractmethod def __len__(self) -> int: @@ -120,7 +123,7 @@ def count(self, **kwargs: Any) -> int: def find( self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] - ) -> Tuple[Union[List[EntryResource], EntryResource, None], int, bool, Set[str]]: + ) -> Tuple[Union[List[EntryResource], EntryResource], int, bool, Set[str]]: """ Fetches results and indicates if more data is available. @@ -133,27 +136,27 @@ def find( Returns: A tuple of various relevant values: - (`results`, `data_returned`, `more_data_available`, `exclude_fields`, `include_fields`). + (`results`, `data_returned`, `more_data_available`, `exclude_fields`). """ criteria = self.handle_query_params(params) single_entry = isinstance(params, SingleEntryQueryParams) response_fields = criteria.pop("fields") - results, data_returned, more_data_available = self._run_db_query( + raw_results, data_returned, more_data_available = self._run_db_query( criteria, single_entry ) exclude_fields = self.all_fields - response_fields - results = [self.resource_mapper.map_back(doc) for doc in results] + results = [self.resource_mapper.map_back(doc) for doc in raw_results] self.check_and_add_missing_fields(results, response_fields) if results: results = self.resource_mapper.deserialize(results) if single_entry: - results = results[0] if results else None + results = results[0] if results else None # type: ignore[assignment] if data_returned > 1: raise NotFound( @@ -260,6 +263,7 @@ def get_non_optional_fields(self) -> List[str]: next_key = path.pop(0) attributes = attributes[next_key] return attributes["required"] + return [] @lru_cache(maxsize=4) def get_attribute_fields(self) -> Set[str]: @@ -323,7 +327,7 @@ def handle_query_params( # filter if getattr(params, "filter", False): cursor_kwargs["filter"] = self.transformer.transform( - self.parser.parse(params.filter) + self.parser.parse(params.filter) # type: ignore[union-attr] ) else: cursor_kwargs["filter"] = {} @@ -339,7 +343,7 @@ def handle_query_params( # page_limit if getattr(params, "page_limit", False): - limit = params.page_limit + limit = params.page_limit # type: ignore[union-attr] if limit > CONFIG.page_limit_max: raise Forbidden( detail=f"Max allowed page_limit is {CONFIG.page_limit_max}, you requested {limit}", @@ -363,7 +367,7 @@ def handle_query_params( # sort if getattr(params, "sort", False): - cursor_kwargs["sort"] = self.parse_sort_params(params.sort) + cursor_kwargs["sort"] = self.parse_sort_params(params.sort) # type: ignore[union-attr] # warn if both page_offset and page_number are given if getattr(params, "page_offset", False): @@ -373,23 +377,23 @@ def handle_query_params( category=QueryParamNotUsed, ) - cursor_kwargs["skip"] = params.page_offset + cursor_kwargs["skip"] = params.page_offset # type: ignore[union-attr] # validate page_number elif isinstance(getattr(params, "page_number", None), int): - if params.page_number < 1: + if params.page_number < 1: # type: ignore[union-attr] warnings.warn( - message=f"'page_number' is 1-based, using 'page_number=1' instead of {params.page_number}", + message=f"'page_number' is 1-based, using 'page_number=1' instead of {params.page_number}", # type: ignore[union-attr] category=QueryParamNotUsed, ) page_number = 1 else: - page_number = params.page_number + page_number = params.page_number # type: ignore[union-attr] cursor_kwargs["skip"] = (page_number - 1) * cursor_kwargs["limit"] return cursor_kwargs - def parse_sort_params(self, sort_params: str) -> Tuple[Tuple[str, int]]: + def parse_sort_params(self, sort_params: str) -> Iterable[Tuple[str, int]]: """Handles any sort parameters passed to the collection, resolving aliases and dealing with any invalid fields. @@ -397,11 +401,11 @@ def parse_sort_params(self, sort_params: str) -> Tuple[Tuple[str, int]]: BadRequest: if an invalid sort is requested. Returns: - A tuple of tuples containing the aliased field name and + A list of tuples containing the aliased field name and sort direction encoded as 1 (ascending) or -1 (descending). """ - sort_spec = [] + sort_spec: List[Tuple[str, int]] = [] for field in sort_params.split(","): sort_dir = 1 if field.startswith("-"): @@ -438,10 +442,10 @@ def parse_sort_params(self, sort_params: str) -> Tuple[Tuple[str, int]]: raise BadRequest(detail=error_detail) # If at least one valid field has been provided for sorting, then use that - sort_spec = tuple( + sort_spec = [ (field, sort_dir) for field, sort_dir in sort_spec if field not in unknown_fields - ) + ] return sort_spec diff --git a/optimade/server/entry_collections/mongo.py b/optimade/server/entry_collections/mongo.py index be1c047774..17a5bd07ab 100644 --- a/optimade/server/entry_collections/mongo.py +++ b/optimade/server/entry_collections/mongo.py @@ -1,4 +1,4 @@ -from typing import Dict, Tuple, List, Any, Union +from typing import Any, Dict, List, Tuple, Type, Union from optimade.filterparser import LarkParser from optimade.filtertransformers.mongo import MongoTransformer @@ -7,8 +7,7 @@ from optimade.server.entry_collections import EntryCollection from optimade.server.logger import LOGGER from optimade.server.mappers import BaseResourceMapper -from optimade.server.query_params import SingleEntryQueryParams, EntryListingQueryParams - +from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams if CONFIG.database_backend.value == "mongodb": from pymongo import MongoClient, version_tuple @@ -39,8 +38,8 @@ class MongoCollection(EntryCollection): def __init__( self, name: str, - resource_cls: EntryResource, - resource_mapper: BaseResourceMapper, + resource_cls: Type[EntryResource], + resource_mapper: Type[BaseResourceMapper], database: str = CONFIG.mongo_database, ): """Initialize the MongoCollection for the given parameters. diff --git a/optimade/server/exception_handlers.py b/optimade/server/exception_handlers.py index 42c358f918..06fc083a85 100644 --- a/optimade/server/exception_handlers.py +++ b/optimade/server/exception_handlers.py @@ -1,26 +1,24 @@ import traceback -from typing import List, Tuple, Callable +from typing import Callable, Iterable, List, Optional, Tuple, Type, Union -from lark.exceptions import VisitError - -from pydantic import ValidationError +from fastapi import Request from fastapi.encoders import jsonable_encoder from fastapi.exceptions import RequestValidationError, StarletteHTTPException -from fastapi import Request - -from optimade.models import OptimadeError, ErrorResponse, ErrorSource +from lark.exceptions import VisitError +from pydantic import ValidationError +from optimade.exceptions import BadRequest, OptimadeHTTPException +from optimade.models import ErrorResponse, ErrorSource, OptimadeError from optimade.server.config import CONFIG -from optimade.server.exceptions import BadRequest from optimade.server.logger import LOGGER -from optimade.server.routers.utils import meta_values, JSONAPIResponse +from optimade.server.routers.utils import JSONAPIResponse, meta_values def general_exception( request: Request, exc: Exception, status_code: int = 500, # A status_code in `exc` will take precedence - errors: List[OptimadeError] = None, + errors: Optional[List[OptimadeError]] = None, ) -> JSONAPIResponse: """Handle an exception @@ -44,17 +42,17 @@ def general_exception( debug_info[f"_{CONFIG.provider.prefix}_traceback"] = tb try: - http_response_code = int(exc.status_code) + http_response_code = int(exc.status_code) # type: ignore[attr-defined] except AttributeError: http_response_code = int(status_code) try: - title = str(exc.title) + title = str(exc.title) # type: ignore[attr-defined] except AttributeError: title = str(exc.__class__.__name__) try: - detail = str(exc.detail) + detail = str(exc.detail) # type: ignore[attr-defined] except AttributeError: detail = str(exc) @@ -80,7 +78,8 @@ def general_exception( def http_exception_handler( - request: Request, exc: StarletteHTTPException + request: Request, + exc: Union[StarletteHTTPException, OptimadeHTTPException], ) -> JSONAPIResponse: """Handle a general HTTP Exception from Starlette @@ -154,7 +153,7 @@ def grammar_not_implemented_handler( All errors raised during filter transformation are wrapped in the Lark `VisitError`. According to the OPTIMADE specification, these errors are repurposed to be 501 NotImplementedErrors. - For special exceptions, like [`BadRequest`][optimade.server.exceptions.BadRequest], we pass-through to + For special exceptions, like [`BadRequest`][optimade.exceptions.BadRequest], we pass-through to [`general_exception()`][optimade.server.exception_handlers.general_exception], since they should not return a 501 NotImplementedError. @@ -221,16 +220,20 @@ def general_exception_handler(request: Request, exc: Exception) -> JSONAPIRespon return general_exception(request, exc) -OPTIMADE_EXCEPTIONS: Tuple[ - Exception, Callable[[Request, Exception], JSONAPIResponse] -] = ( +OPTIMADE_EXCEPTIONS: Iterable[ + Tuple[ + Type[Exception], + Callable[[Request, Exception], JSONAPIResponse], + ] +] = [ (StarletteHTTPException, http_exception_handler), + (OptimadeHTTPException, http_exception_handler), (RequestValidationError, request_validation_exception_handler), (ValidationError, validation_exception_handler), (VisitError, grammar_not_implemented_handler), - (NotImplementedError, not_implemented_handler), + (NotImplementedError, not_implemented_handler), # type: ignore[list-item] # not entirely sure why this entry triggers mypy (Exception, general_exception_handler), -) +] """A tuple of all pairs of exceptions and handler functions that allow for appropriate responses to be returned in certain scenarios according to the OPTIMADE specification. diff --git a/optimade/server/exceptions.py b/optimade/server/exceptions.py index c4daf9eaf6..995cfd8c87 100644 --- a/optimade/server/exceptions.py +++ b/optimade/server/exceptions.py @@ -1,101 +1,25 @@ -from abc import ABC -from fastapi import HTTPException as FastAPIHTTPException +"""Reproduced imports from `optimade.exceptions` for backwards-compatibility.""" + +from optimade.exceptions import ( + POSSIBLE_ERRORS, + BadRequest, + Forbidden, + InternalServerError, + NotFound, + NotImplementedResponse, + OptimadeHTTPException, + UnprocessableEntity, + VersionNotSupported, +) __all__ = ( + "OptimadeHTTPException", "BadRequest", "VersionNotSupported", "Forbidden", "NotFound", "UnprocessableEntity", + "InternalServerError", "NotImplementedResponse", "POSSIBLE_ERRORS", ) - - -class HTTPException(FastAPIHTTPException, ABC): - """This abstract class makes it easier to subclass FastAPI's HTTPException with - new status codes. - - It can also be useful when testing requires a string representation - of an exception that contains the HTTPException - detail string, rather than the standard Python exception message. - - Attributes: - status_code: The HTTP status code accompanying this exception. - title: A descriptive title for this exception. - - """ - - status_code: int = None - title: str - - def __init__(self, detail: str = None, headers: dict = None) -> None: - if self.status_code is None: - raise AttributeError( - "HTTPException class {self.__class__.__name__} is missing required `status_code` attribute." - ) - - super().__init__(status_code=self.status_code, detail=detail, headers=headers) - - def __str__(self) -> str: - return self.detail if self.detail is not None else self.__repr__() - - -class BadRequest(HTTPException): - """400 Bad Request""" - - status_code: int = 400 - title: str = "Bad Request" - - -class VersionNotSupported(HTTPException): - """553 Version Not Supported""" - - status_code: int = 553 - title: str = "Version Not Supported" - - -class Forbidden(HTTPException): - """403 Forbidden""" - - status_code: int = 403 - title: str = "Forbidden" - - -class NotFound(HTTPException): - """404 Not Found""" - - status_code: int = 404 - title: str = "Not Found" - - -class UnprocessableEntity(HTTPException): - """422 Unprocessable Entity""" - - status_code: int = 422 - title: str = "Unprocessable Entity" - - -class InternalServerError(HTTPException): - """500 Internal Server Error""" - - status_code: int = 500 - title: str = "Internal Server Error" - - -class NotImplementedResponse(HTTPException): - """501 Not Implemented""" - - status_code: int = 501 - title: str = "Not Implemented" - - -POSSIBLE_ERRORS = ( - BadRequest, - Forbidden, - NotFound, - UnprocessableEntity, - InternalServerError, - NotImplementedResponse, - VersionNotSupported, -) diff --git a/optimade/server/logger.py b/optimade/server/logger.py index 264e22ce86..cae1ec35c0 100644 --- a/optimade/server/logger.py +++ b/optimade/server/logger.py @@ -1,9 +1,9 @@ """Logging to both file and terminal""" import logging +import logging.handlers import os -from pathlib import Path import sys - +from pathlib import Path # Instantiate LOGGER LOGGER = logging.getLogger("optimade") diff --git a/optimade/server/main.py b/optimade/server/main.py index 48a534e353..4f95dcccb5 100644 --- a/optimade/server/main.py +++ b/optimade/server/main.py @@ -18,10 +18,9 @@ from optimade import __api_version__, __version__ from optimade.server.entry_collections import EntryCollection -from optimade.server.logger import LOGGER from optimade.server.exception_handlers import OPTIMADE_EXCEPTIONS +from optimade.server.logger import LOGGER from optimade.server.middleware import OPTIMADE_MIDDLEWARE - from optimade.server.routers import ( info, landing, @@ -64,6 +63,7 @@ if CONFIG.insert_test_data: import bson.json_util from bson.objectid import ObjectId + import optimade.server.data as data from optimade.server.routers import ENTRY_COLLECTIONS from optimade.server.routers.utils import get_providers @@ -81,7 +81,7 @@ def load_entries(endpoint_name: str, endpoint_collection: EntryCollection): ) providers = get_providers(add_mongo_id=True) for doc in providers: - endpoint_collection.collection.replace_one( + endpoint_collection.collection.replace_one( # type: ignore[attr-defined] filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads(bson.json_util.dumps(doc)), upsert=True, diff --git a/optimade/server/main_index.py b/optimade/server/main_index.py index a50bafed7b..d57b71877c 100644 --- a/optimade/server/main_index.py +++ b/optimade/server/main_index.py @@ -18,8 +18,8 @@ config_warnings = w from optimade import __api_version__, __version__ -from optimade.server.logger import LOGGER from optimade.server.exception_handlers import OPTIMADE_EXCEPTIONS +from optimade.server.logger import LOGGER from optimade.server.middleware import OPTIMADE_MIDDLEWARE from optimade.server.routers import index_info, links, versions from optimade.server.routers.utils import BASE_URL_PREFIXES, JSONAPIResponse @@ -59,8 +59,9 @@ if CONFIG.insert_test_data and CONFIG.index_links_path.exists(): import bson.json_util from bson.objectid import ObjectId + from optimade.server.routers.links import links_coll - from optimade.server.routers.utils import mongo_id_for_database, get_providers + from optimade.server.routers.utils import get_providers, mongo_id_for_database LOGGER.debug("Loading index links...") with open(CONFIG.index_links_path) as f: @@ -83,7 +84,7 @@ ) providers = get_providers(add_mongo_id=True) for doc in providers: - links_coll.collection.replace_one( + links_coll.collection.replace_one( # type: ignore[attr-defined] filter={"_id": ObjectId(doc["_id"]["$oid"])}, replacement=bson.json_util.loads(bson.json_util.dumps(doc)), upsert=True, diff --git a/optimade/server/mappers/__init__.py b/optimade/server/mappers/__init__.py index 63e2656493..c38e6ccd08 100644 --- a/optimade/server/mappers/__init__.py +++ b/optimade/server/mappers/__init__.py @@ -5,8 +5,8 @@ from .structures import * # noqa: F403 __all__ = ( - entries.__all__ # noqa: F405 - + links.__all__ # noqa: F405 - + references.__all__ # noqa: F405 - + structures.__all__ # noqa: F405 + entries.__all__ # type: ignore[name-defined] # noqa: F405 + + links.__all__ # type: ignore[name-defined] # noqa: F405 + + references.__all__ # type: ignore[name-defined] # noqa: F405 + + structures.__all__ # type: ignore[name-defined] # noqa: F405 ) diff --git a/optimade/server/mappers/entries.py b/optimade/server/mappers/entries.py index 9818828c8f..a25e93ff2d 100644 --- a/optimade/server/mappers/entries.py +++ b/optimade/server/mappers/entries.py @@ -1,12 +1,13 @@ -from typing import Tuple, Optional, Type, Set, Dict, Any, List, Iterable -from functools import lru_cache import warnings -from optimade.server.config import CONFIG +from functools import lru_cache +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type + from optimade.models.entries import EntryResource +from optimade.server.config import CONFIG from optimade.utils import ( - write_to_nested_dict, read_from_nested_dict, remove_from_nested_dict, + write_to_nested_dict, ) __all__ = ("BaseResourceMapper",) @@ -61,25 +62,19 @@ class BaseResourceMapper: """ try: - from optimade.server.data import ( - providers as PROVIDERS, - ) # pylint: disable=no-name-in-module + from optimade.server.data import providers as PROVIDERS # type: ignore except (ImportError, ModuleNotFoundError): PROVIDERS = {} KNOWN_PROVIDER_PREFIXES: Set[str] = set( prov["id"] for prov in PROVIDERS.get("data", []) ) - ALIASES: Tuple[Tuple[str, str]] = () - LENGTH_ALIASES: Tuple[Tuple[str, str]] = () - PROVIDER_FIELDS: Tuple[str] = () + ALIASES: Tuple[Tuple[str, str], ...] = () + LENGTH_ALIASES: Tuple[Tuple[str, str], ...] = () + PROVIDER_FIELDS: Tuple[str, ...] = () ENTRY_RESOURCE_CLASS: Type[EntryResource] = EntryResource RELATIONSHIP_ENTRY_TYPES: Set[str] = {"references", "structures"} TOP_LEVEL_NON_ATTRIBUTES_FIELDS: Set[str] = {"id", "type", "relationships", "links"} - SUPPORTED_PREFIXES: Set[str] - ALL_ATTRIBUTES: Set[str] - ENTRY_RESOURCE_ATTRIBUTES: Dict[str, Any] - ENDPOINT: str @classmethod @lru_cache(maxsize=1) @@ -196,7 +191,7 @@ def ENDPOINT(cls) -> str: @classmethod @lru_cache(maxsize=1) - def all_length_aliases(cls) -> Iterable[Tuple[str, str]]: + def all_length_aliases(cls) -> Tuple[Tuple[str, str], ...]: """Returns all of the associated length aliases for this class, including those defined by the server config. @@ -225,7 +220,7 @@ def length_alias_for(cls, field: str) -> Optional[str]: return dict(cls.all_length_aliases()).get(field, None) @classmethod - def get_map_field_from_dict(cls, field: str, aliases: dict): + def get_map_field_from_dict(cls, field: str, aliases: Dict[str, str]) -> str: """Replaces (part of) the field_name "field" with the matching field in the dictionary dict" It first tries to find the entire field name(incl. subfields(which are separated by:".")) in the dictionary. If it is not present it removes the deepest nesting level and checks again. @@ -236,10 +231,10 @@ def get_map_field_from_dict(cls, field: str, aliases: dict): for i in range(len(split), 0, -1): field_path = ".".join(split[0:i]) if field_path in aliases: - field = aliases.get(field_path) + field_alias: str = aliases[field_path] if split[i:]: - field += "." + ".".join(split[i:]) - break + field_alias += "." + ".".join(split[i:]) + return field_alias return field @classmethod @@ -417,7 +412,7 @@ def add_alias_and_prefix(cls, doc: dict) -> dict: Returns: A dictionary with the fieldnames as presented by OPTIMADE """ - newdoc = {} + newdoc: dict = {} mod_doc = doc.copy() # First apply all the aliases (They are sorted so the deepest nesting level occurs first.) sorted_aliases = sorted(cls.all_aliases(), key=lambda ele: ele[0], reverse=True) diff --git a/optimade/server/mappers/links.py b/optimade/server/mappers/links.py index cba0e28b23..bb3a531e45 100644 --- a/optimade/server/mappers/links.py +++ b/optimade/server/mappers/links.py @@ -1,13 +1,11 @@ -from optimade.server.mappers.entries import BaseResourceMapper from optimade.models.links import LinksResource +from optimade.server.mappers.entries import BaseResourceMapper __all__ = ("LinksMapper",) class LinksMapper(BaseResourceMapper): - ENDPOINT = "links" - ENTRY_RESOURCE_CLASS = LinksResource @classmethod diff --git a/optimade/server/mappers/references.py b/optimade/server/mappers/references.py index e756296e08..c5a5c858ea 100644 --- a/optimade/server/mappers/references.py +++ b/optimade/server/mappers/references.py @@ -1,5 +1,5 @@ -from optimade.server.mappers.entries import BaseResourceMapper from optimade.models.references import ReferenceResource +from optimade.server.mappers.entries import BaseResourceMapper __all__ = ("ReferenceMapper",) diff --git a/optimade/server/mappers/structures.py b/optimade/server/mappers/structures.py index a63bb171ff..0b0826f85a 100644 --- a/optimade/server/mappers/structures.py +++ b/optimade/server/mappers/structures.py @@ -1,5 +1,5 @@ -from optimade.server.mappers.entries import BaseResourceMapper from optimade.models.structures import StructureResource +from optimade.server.mappers.entries import BaseResourceMapper __all__ = ("StructureMapper",) diff --git a/optimade/server/middleware.py b/optimade/server/middleware.py index b25642bd49..ed9a69dc2a 100644 --- a/optimade/server/middleware.py +++ b/optimade/server/middleware.py @@ -5,31 +5,27 @@ See the specific Starlette [documentation page](https://www.starlette.io/middleware/) for more information on it's middleware implementation. """ +import json import re -from typing import Optional, IO, Type, Generator, List, Union, Tuple import urllib.parse import warnings - -try: - import simplejson as json -except ImportError: - import json +from typing import Generator, Iterable, List, Optional, TextIO, Type, Union from starlette.datastructures import URL as StarletteURL from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import RedirectResponse, StreamingResponse +from optimade.exceptions import BadRequest, VersionNotSupported from optimade.models import Warnings from optimade.server.config import CONFIG -from optimade.server.exceptions import BadRequest, VersionNotSupported -from optimade.server.warnings import ( +from optimade.server.routers.utils import BASE_URL_PREFIXES, get_base_url +from optimade.warnings import ( FieldValueNotRecognized, OptimadeWarning, QueryParamNotUsed, TooManyValues, ) -from optimade.server.routers.utils import get_base_url, BASE_URL_PREFIXES class EnsureQueryParamIntegrity(BaseHTTPMiddleware): @@ -156,9 +152,8 @@ def handle_api_hint(api_hint: List[str]) -> Union[None, str]: for value in api_hint: values = value.split(",") _api_hint.extend(values) - api_hint = _api_hint - if len(api_hint) > 1: + if len(_api_hint) > 1: warnings.warn( TooManyValues( detail="`api_hint` should only be supplied once, with a single value." @@ -166,29 +161,21 @@ def handle_api_hint(api_hint: List[str]) -> Union[None, str]: ) return None - api_hint = f"/{api_hint[0]}" - if re.match(r"^/v[0-9]+(\.[0-9]+)?$", api_hint) is None: + api_hint_str: str = f"/{_api_hint[0]}" + if re.match(r"^/v[0-9]+(\.[0-9]+)?$", api_hint_str) is None: warnings.warn( FieldValueNotRecognized( - detail=f"{api_hint[1:]!r} is not recognized as a valid `api_hint` value." + detail=f"{api_hint_str[1:]!r} is not recognized as a valid `api_hint` value." ) ) return None - if api_hint in BASE_URL_PREFIXES.values(): - return api_hint + if api_hint_str in BASE_URL_PREFIXES.values(): + return api_hint_str - major_api_hint = int(re.findall(r"/v([0-9]+)", api_hint)[0]) + major_api_hint = int(re.findall(r"/v([0-9]+)", api_hint_str)[0]) major_implementation = int(BASE_URL_PREFIXES["major"][len("/v") :]) - if major_api_hint > major_implementation: - # Let's not try to handle a request for a newer major version - raise VersionNotSupported( - detail=( - f"The provided `api_hint` ({api_hint[1:]!r}) is not supported by this implementation. " - f"Supported versions include: {', '.join(BASE_URL_PREFIXES.values())}" - ) - ) if major_api_hint <= major_implementation: # If less than: # Use the current implementation in hope that it can still handle older requests @@ -197,6 +184,14 @@ def handle_api_hint(api_hint: List[str]) -> Union[None, str]: # Go to /v, since this should point to the latest available return BASE_URL_PREFIXES["major"] + # Let's not try to handle a request for a newer major version + raise VersionNotSupported( + detail=( + f"The provided `api_hint` ({api_hint_str[1:]!r}) is not supported by this implementation. " + f"Supported versions include: {', '.join(BASE_URL_PREFIXES.values())}" + ) + ) + @staticmethod def is_versioned_base_url(url: str) -> bool: """Determine whether a request is for a versioned base URL. @@ -247,18 +242,18 @@ async def dispatch(self, request: Request, call_next): f"{base_url}{version_path}{str(request.url)[len(base_url):]}" ) url = urllib.parse.urlsplit(new_request) - parsed_query = urllib.parse.parse_qsl( - url.query, keep_blank_values=True - ) - parsed_query = "&".join( + q = "&".join( [ f"{key}={value}" - for key, value in parsed_query + for key, value in urllib.parse.parse_qsl( + url.query, keep_blank_values=True + ) if key != "api_hint" ] ) + return RedirectResponse( - request.url.replace(path=url.path, query=parsed_query), + request.url.replace(path=url.path, query=q), headers=request.headers, ) # This is the non-URL changing solution: @@ -273,9 +268,9 @@ async def dispatch(self, request: Request, call_next): class AddWarnings(BaseHTTPMiddleware): """ - Add [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning]s to the response. + Add [`OptimadeWarning`][optimade.warnings.OptimadeWarning]s to the response. - All sub-classes of [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] + All sub-classes of [`OptimadeWarning`][optimade.warnings.OptimadeWarning] will also be added to the response's [`meta.warnings`][optimade.models.optimade_json.ResponseMeta.warnings] list. @@ -312,13 +307,15 @@ class AddWarnings(BaseHTTPMiddleware): """ + _warnings: List[Warnings] + def showwarning( self, - message: Warning, + message: Union[Warning, str], category: Type[Warning], filename: str, lineno: int, - file: Optional[IO] = None, + file: Optional[TextIO] = None, line: Optional[str] = None, ) -> None: """ @@ -335,12 +332,12 @@ def showwarning( This method will also print warning messages to `stderr` by calling `warnings._showwarning_orig()` or `warnings._showwarnmsg_impl()`. The first function will be called if the issued warning is not recognized - as an [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning]. + as an [`OptimadeWarning`][optimade.warnings.OptimadeWarning]. This is equivalent to "standard behaviour". The second function will be called _after_ an - [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] has been handled. + [`OptimadeWarning`][optimade.warnings.OptimadeWarning] has been handled. - An [`OptimadeWarning`][optimade.server.warnings.OptimadeWarning] will be + An [`OptimadeWarning`][optimade.warnings.OptimadeWarning] will be translated into an OPTIMADE Warnings JSON object in accordance with [the specification](https://github.com/Materials-Consortia/OPTIMADE/blob/v1.0.0/optimade.rst#json-response-schema-common-fields). This process is similar to the [Exception handlers][optimade.server.exception_handlers]. @@ -361,7 +358,7 @@ def showwarning( if not isinstance(message, OptimadeWarning): # If the Warning is not an OptimadeWarning or subclass thereof, # use the regular 'showwarning' function. - warnings._showwarning_orig(message, category, filename, lineno, file, line) + warnings._showwarning_orig(message, category, filename, lineno, file, line) # type: ignore[attr-defined] return # Format warning @@ -387,7 +384,7 @@ def showwarning( # When a warning is logged during Python shutdown, linecache # and the import machinery don't work anymore line = None - linecache = None + linecache = None # type: ignore[assignment] meta = { "filename": filename, "lineno": lineno, @@ -404,16 +401,16 @@ def showwarning( self._warnings.append(new_warning.dict(exclude_unset=True)) # Show warning message as normal in sys.stderr - warnings._showwarnmsg_impl( + warnings._showwarnmsg_impl( # type: ignore[attr-defined] warnings.WarningMessage(message, category, filename, lineno, file, line) ) @staticmethod - def chunk_it_up(content: str, chunk_size: int) -> Generator: + def chunk_it_up(content: Union[str, bytes], chunk_size: int) -> Generator: """Return generator for string in chunks of size `chunk_size`. Parameters: - content: String-content to separate into chunks. + content: String or bytes content to separate into chunks. chunk_size: The size of the chunks, i.e. the length of the string-chunks. Returns: @@ -445,17 +442,17 @@ async def dispatch(self, request: Request, call_next): if not isinstance(chunk, bytes): chunk = chunk.encode(charset) body += chunk - body = body.decode(charset) + body_str = body.decode(charset) if self._warnings: - response = json.loads(body) + response = json.loads(body_str) response.get("meta", {})["warnings"] = self._warnings - body = json.dumps(response) + body_str = json.dumps(response) if "content-length" in headers: - headers["content-length"] = str(len(body)) + headers["content-length"] = str(len(body_str)) response = StreamingResponse( - content=self.chunk_it_up(body, chunk_size), + content=self.chunk_it_up(body_str, chunk_size), status_code=status, headers=headers, media_type=media_type, @@ -465,7 +462,7 @@ async def dispatch(self, request: Request, call_next): return response -OPTIMADE_MIDDLEWARE: Tuple[BaseHTTPMiddleware] = ( +OPTIMADE_MIDDLEWARE: Iterable[BaseHTTPMiddleware] = ( EnsureQueryParamIntegrity, CheckWronglyVersionedBaseUrls, HandleApiHint, diff --git a/optimade/server/query_params.py b/optimade/server/query_params.py index 9796bb814b..71cfd86b6b 100644 --- a/optimade/server/query_params.py +++ b/optimade/server/query_params.py @@ -1,12 +1,14 @@ +from abc import ABC +from typing import Iterable, List +from warnings import warn + from fastapi import Query from pydantic import EmailStr # pylint: disable=no-name-in-module -from typing import Iterable, List + +from optimade.exceptions import BadRequest from optimade.server.config import CONFIG -from warnings import warn from optimade.server.mappers import BaseResourceMapper -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import UnknownProviderQueryParameter, QueryParamNotUsed -from abc import ABC +from optimade.warnings import QueryParamNotUsed, UnknownProviderQueryParameter class BaseQueryParams(ABC): diff --git a/optimade/server/routers/index_info.py b/optimade/server/routers/index_info.py index 6779220f89..506a802963 100644 --- a/optimade/server/routers/index_info.py +++ b/optimade/server/routers/index_info.py @@ -2,17 +2,16 @@ from optimade import __api_version__ from optimade.models import ( - IndexInfoResponse, IndexInfoAttributes, IndexInfoResource, + IndexInfoResponse, IndexRelationship, RelatedLinksResource, ) from optimade.server.config import CONFIG -from optimade.server.routers.utils import meta_values, get_base_url +from optimade.server.routers.utils import get_base_url, meta_values from optimade.server.schemas import ERROR_RESPONSES - router = APIRouter(redirect_slashes=True) diff --git a/optimade/server/routers/info.py b/optimade/server/routers/info.py index ddd48adfdd..74c96ccd57 100644 --- a/optimade/server/routers/info.py +++ b/optimade/server/routers/info.py @@ -2,16 +2,15 @@ from fastapi.exceptions import StarletteHTTPException from optimade import __api_version__ -from optimade.models import InfoResponse, EntryInfoResponse -from optimade.server.routers.utils import meta_values, get_base_url +from optimade.models import EntryInfoResponse, InfoResponse from optimade.server.config import CONFIG +from optimade.server.routers.utils import get_base_url, meta_values from optimade.server.schemas import ( ENTRY_INFO_SCHEMAS, ERROR_RESPONSES, retrieve_queryable_properties, ) - router = APIRouter(redirect_slashes=True) @@ -23,7 +22,7 @@ responses=ERROR_RESPONSES, ) def get_info(request: Request) -> InfoResponse: - from optimade.models import BaseInfoResource, BaseInfoAttributes + from optimade.models import BaseInfoAttributes, BaseInfoResource return InfoResponse( meta=meta_values( diff --git a/optimade/server/routers/landing.py b/optimade/server/routers/landing.py index e4b5d95ba1..151fa85b22 100644 --- a/optimade/server/routers/landing.py +++ b/optimade/server/routers/landing.py @@ -1,16 +1,16 @@ """ OPTIMADE landing page router. """ -from pathlib import Path from functools import lru_cache +from pathlib import Path from fastapi import Request from fastapi.responses import HTMLResponse -from starlette.routing import Router, Route -from optimade import __api_version__ +from starlette.routing import Route, Router -from optimade.server.routers import ENTRY_COLLECTIONS -from optimade.server.routers.utils import meta_values, get_base_url +from optimade import __api_version__ from optimade.server.config import CONFIG +from optimade.server.routers import ENTRY_COLLECTIONS +from optimade.server.routers.utils import get_base_url, meta_values @lru_cache() diff --git a/optimade/server/routers/links.py b/optimade/server/routers/links.py index 73852d2a4f..e09e503de9 100644 --- a/optimade/server/routers/links.py +++ b/optimade/server/routers/links.py @@ -1,6 +1,6 @@ from fastapi import APIRouter, Depends, Request -from optimade.models import LinksResponse, LinksResource +from optimade.models import LinksResource, LinksResponse from optimade.server.config import CONFIG from optimade.server.entry_collections import create_collection from optimade.server.mappers import LinksMapper diff --git a/optimade/server/routers/references.py b/optimade/server/routers/references.py index d6fb9b0a84..ca39582b52 100644 --- a/optimade/server/routers/references.py +++ b/optimade/server/routers/references.py @@ -12,7 +12,6 @@ from optimade.server.routers.utils import get_entries, get_single_entry from optimade.server.schemas import ERROR_RESPONSES - router = APIRouter(redirect_slashes=True) references_coll = create_collection( diff --git a/optimade/server/routers/utils.py b/optimade/server/routers/utils.py index 91f8cd3caa..761b6f9a16 100644 --- a/optimade/server/routers/utils.py +++ b/optimade/server/routers/utils.py @@ -2,26 +2,25 @@ import re import urllib.parse from datetime import datetime -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Dict, List, Optional, Set, Type, Union from fastapi import Request from fastapi.responses import JSONResponse from starlette.datastructures import URL as StarletteURL from optimade import __api_version__ +from optimade.exceptions import BadRequest, InternalServerError from optimade.models import ( - ResponseMeta, EntryResource, EntryResponseMany, EntryResponseOne, + ResponseMeta, ToplevelLinks, ) - from optimade.server.config import CONFIG from optimade.server.entry_collections import EntryCollection -from optimade.server.exceptions import BadRequest, InternalServerError from optimade.server.query_params import EntryListingQueryParams, SingleEntryQueryParams -from optimade.utils import mongo_id_for_database, get_providers, PROVIDER_LIST_URLS +from optimade.utils import PROVIDER_LIST_URLS, get_providers, mongo_id_for_database __all__ = ( "BASE_URL_PREFIXES", @@ -129,7 +128,7 @@ def get_included_relationships( results: Union[EntryResource, List[EntryResource]], ENTRY_COLLECTIONS: Dict[str, EntryCollection], include_param: List[str], -) -> Dict[str, List[EntryResource]]: +) -> List[Union[EntryResource, Dict]]: """Filters the included relationships and makes the appropriate compound request to include them in the response. @@ -157,7 +156,7 @@ def get_included_relationships( f"Known relationship types: {sorted(ENTRY_COLLECTIONS.keys())}" ) - endpoint_includes = defaultdict(dict) + endpoint_includes: Dict[Any, Dict] = defaultdict(dict) for doc in results: # convert list of references into dict by ID to only included unique IDs if doc is None: @@ -188,8 +187,8 @@ def get_included_relationships( params = EntryListingQueryParams( filter=compound_filter, response_format="json", - response_fields=None, - sort=None, + response_fields="", + sort="", page_limit=0, page_offset=0, ) @@ -225,7 +224,7 @@ def get_base_url( def get_entries( collection: EntryCollection, - response: EntryResponseMany, + response: Type[EntryResponseMany], request: Request, params: EntryListingQueryParams, ) -> EntryResponseMany: @@ -243,12 +242,16 @@ def get_entries( include = [] if getattr(params, "include", False): include.extend(params.include.split(",")) - included = get_included_relationships(results, ENTRY_COLLECTIONS, include) + + included = [] + if results is not None: + included = get_included_relationships(results, ENTRY_COLLECTIONS, include) if more_data_available: # Deduce the `next` link from the current request query = urllib.parse.parse_qs(request.url.query) - query["page_offset"] = int(query.get("page_offset", [0])[0]) + len(results) + if isinstance(results, list): + query["page_offset"] = int(query.get("page_offset", [0])[0]) + len(results) # type: ignore[assignment,list-item] urlencoded = urllib.parse.urlencode(query, doseq=True) base_url = get_base_url(request.url) @@ -278,14 +281,14 @@ def get_entries( def get_single_entry( collection: EntryCollection, entry_id: str, - response: EntryResponseOne, + response: Type[EntryResponseOne], request: Request, params: SingleEntryQueryParams, ) -> EntryResponseOne: from optimade.server.routers import ENTRY_COLLECTIONS params.check_params(request.query_params) - params.filter = f'id="{entry_id}"' + params.filter = f'id="{entry_id}"' # type: ignore[attr-defined] ( results, data_returned, @@ -293,16 +296,19 @@ def get_single_entry( exclude_fields, ) = collection.find(params) - include = [] - if getattr(params, "include", False): - include.extend(params.include.split(",")) - included = get_included_relationships(results, ENTRY_COLLECTIONS, include) - if more_data_available: raise InternalServerError( detail=f"more_data_available MUST be False for single entry response, however it is {more_data_available}", ) + include = [] + if getattr(params, "include", False): + include.extend(params.include.split(",")) + + included = [] + if results is not None: + included = get_included_relationships(results, ENTRY_COLLECTIONS, include) + links = ToplevelLinks(next=None) if exclude_fields and results is not None: diff --git a/optimade/server/routers/versions.py b/optimade/server/routers/versions.py index 9e4c315919..5d2ebae88f 100644 --- a/optimade/server/routers/versions.py +++ b/optimade/server/routers/versions.py @@ -1,4 +1,4 @@ -from fastapi import Request, APIRouter +from fastapi import APIRouter, Request from fastapi.responses import Response from optimade.server.routers.utils import BASE_URL_PREFIXES diff --git a/optimade/server/schemas.py b/optimade/server/schemas.py index fcb4945061..558745548e 100644 --- a/optimade/server/schemas.py +++ b/optimade/server/schemas.py @@ -1,30 +1,41 @@ -from typing import Dict, Callable +from typing import Callable, Dict, Iterable, Optional + from optimade.models import ( DataType, ErrorResponse, - StructureResource, ReferenceResource, + StructureResource, ) -from optimade.server.exceptions import POSSIBLE_ERRORS __all__ = ("ENTRY_INFO_SCHEMAS", "ERROR_RESPONSES", "retrieve_queryable_properties") -ENTRY_INFO_SCHEMAS: Dict[str, Callable[[None], Dict]] = { +ENTRY_INFO_SCHEMAS: Dict[str, Callable[[], Dict]] = { "structures": StructureResource.schema, "references": ReferenceResource.schema, } """This dictionary is used to define the `/info/` endpoints.""" -ERROR_RESPONSES: Dict[int, Dict] = { - err.status_code: {"model": ErrorResponse, "description": err.title} - for err in POSSIBLE_ERRORS -} +try: + """The possible errors list contains FastAPI/starlette exception objects + This dictionary is only used for constructing the OpenAPI schema, i.e., + a development task, so can be safely nulled to allow other non-server + submodules (e.g., the validator) to access the other schemas + (that only require pydantic to construct). + """ + from optimade.exceptions import POSSIBLE_ERRORS + + ERROR_RESPONSES: Optional[Dict[int, Dict]] = { + err.status_code: {"model": ErrorResponse, "description": err.title} + for err in POSSIBLE_ERRORS + } +except ModuleNotFoundError: + ERROR_RESPONSES = None def retrieve_queryable_properties( schema: dict, - queryable_properties: list = None, - entry_type: str = None, + queryable_properties: Optional[Iterable[str]] = None, + entry_type: Optional[str] = None, ) -> dict: """Recursively loops through the schema of a pydantic model and resolves all references, returning a dictionary of all the @@ -79,7 +90,7 @@ def retrieve_queryable_properties( described_provider_fields = [ field - for field in CONFIG.provider_fields.get(entry_type, {}) + for field in CONFIG.provider_fields.get(entry_type, {}) # type: ignore[call-overload] if isinstance(field, dict) ] for field in described_provider_fields: diff --git a/optimade/server/warnings.py b/optimade/server/warnings.py index 5ad88f95fb..f8ecdb15d1 100644 --- a/optimade/server/warnings.py +++ b/optimade/server/warnings.py @@ -1,62 +1,26 @@ -class OptimadeWarning(Warning): - """Base Warning for the `optimade` package""" - - def __init__(self, detail: str = None, title: str = None, *args) -> None: - detail = detail if detail else self.__doc__ - super().__init__(detail, *args) - self.detail = detail - self.title = title if title else self.__class__.__name__ - - def __repr__(self) -> str: - attrs = {"detail": self.detail, "title": self.title} - return "<{:s}({:s})>".format( - self.__class__.__name__, - " ".join( - [ - f"{attr}={value!r}" - for attr, value in attrs.items() - if value is not None - ] - ), - ) - - def __str__(self) -> str: - return self.detail if self.detail is not None else "" - - -class FieldValueNotRecognized(OptimadeWarning): - """A field or value used in the request is not recognised by this implementation.""" - - -class TooManyValues(OptimadeWarning): - """A field or query parameter has too many values to be handled by this implementation.""" - - -class QueryParamNotUsed(OptimadeWarning): - """A query parameter is not used in this request.""" - - -class MissingExpectedField(OptimadeWarning): - """A field was provided with a null value when a related field was provided - with a value.""" - - -class TimestampNotRFCCompliant(OptimadeWarning): - """A timestamp has been used in a filter that contains microseconds and is thus not - RFC 3339 compliant. This may cause undefined behaviour in the query results. - - """ - - -class UnknownProviderProperty(OptimadeWarning): - """A provider-specific property has been requested via `response_fields` or as in a `filter` that is not - recognised by this implementation. - - """ - - -class UnknownProviderQueryParameter(OptimadeWarning): - """A provider-specific query parameter has been requested in the query with a prefix not - recognised by this implementation. - - """ +"""This submodule maintains backwards compatibility with the old `optimade.server.warnings` module, +which previously implemented the imported warnings directly. + +""" + +from optimade.warnings import ( + FieldValueNotRecognized, + MissingExpectedField, + OptimadeWarning, + QueryParamNotUsed, + TimestampNotRFCCompliant, + TooManyValues, + UnknownProviderProperty, + UnknownProviderQueryParameter, +) + +__all__ = ( + "FieldValueNotRecognized", + "MissingExpectedField", + "OptimadeWarning", + "QueryParamNotUsed", + "TimestampNotRFCCompliant", + "TooManyValues", + "UnknownProviderProperty", + "UnknownProviderQueryParameter", +) diff --git a/optimade/utils.py b/optimade/utils.py index a5baf4d90a..df2e1ba798 100644 --- a/optimade/utils.py +++ b/optimade/utils.py @@ -4,9 +4,10 @@ """ import json -from typing import List, Iterable, Any +from typing import Any, Iterable, List from pydantic import ValidationError + from optimade.models.links import LinksResource PROVIDER_LIST_URLS = ( @@ -46,12 +47,9 @@ def get_providers(add_mongo_id: bool = False) -> list: List of raw JSON-decoded providers including MongoDB object IDs. """ - import requests + import json - try: - import simplejson as json - except ImportError: - import json + import requests for provider_list_url in PROVIDER_LIST_URLS: try: @@ -66,7 +64,7 @@ def get_providers(add_mongo_id: bool = False) -> list: break else: try: - from optimade.server.data import providers + from optimade.server.data import providers # type: ignore except ImportError: from optimade.server.logger import LOGGER @@ -116,12 +114,13 @@ def get_child_database_links( Raises: RuntimeError: If the provider's index meta-database is down, - invalid, or the request otherwise fails. + invalid, or the request otherwise fails. """ import requests + + from optimade.models.links import Aggregate, LinkType from optimade.models.responses import LinksResponse - from optimade.models.links import LinkType, Aggregate base_url = provider.pop("base_url") if base_url is None: @@ -135,23 +134,27 @@ def get_child_database_links( if links.status_code != 200: raise RuntimeError( - f"Invalid response from {links_endp} for provider {provider['id']}: {links.content}." + f"Invalid response from {links_endp} for provider {provider['id']}: {links.content!r}." ) try: - links = LinksResponse(**links.json()) + links_resp = LinksResponse(**links.json()) except (ValidationError, json.JSONDecodeError) as exc: raise RuntimeError( - f"Did not understand response from {provider['id']}: {links.content}" + f"Did not understand response from {provider['id']}: {links.content!r}" ) from exc - return [ - link - for link in links.data - if link.attributes.link_type == LinkType.CHILD - and link.attributes.base_url is not None - and (not obey_aggregate or link.attributes.aggregate == Aggregate.OK) - ] + if isinstance(links_resp.data, LinksResource): + return [ + link + for link in links_resp.data + if link.attributes.link_type == LinkType.CHILD + and link.attributes.base_url is not None + and (not obey_aggregate or link.attributes.aggregate == Aggregate.OK) + ] + + else: + raise RuntimeError("Invalid links responses received: {links.content!r") def get_all_databases() -> Iterable[str]: @@ -238,7 +241,7 @@ def set_field_to_none_if_missing_in_dict(entry: dict, field: str): split_field = field.split(".", 1) # It would be nice if there would be a more universal way to handle special cases like this. if split_field[0] == "structure_features": - value = [] + value: Any = [] else: value = None write_to_nested_dict(entry, field, value) diff --git a/optimade/validator/__init__.py b/optimade/validator/__init__.py index 2e5f3b1678..a98413e28c 100644 --- a/optimade/validator/__init__.py +++ b/optimade/validator/__init__.py @@ -1,19 +1,21 @@ """ This module contains the ImplementationValidator class and corresponding command line tools. """ # pylint: disable=import-outside-toplevel import warnings -from optimade import __version__, __api_version__ -from .validator import ImplementationValidator + +from optimade import __api_version__, __version__ + from .utils import DEFAULT_CONN_TIMEOUT, DEFAULT_READ_TIMEOUT +from .validator import ImplementationValidator __all__ = ["ImplementationValidator", "validate"] def validate(): # pragma: no cover import argparse - import sys + import json import os + import sys import traceback - import json parser = argparse.ArgumentParser( prog="optimade-validator", diff --git a/optimade/validator/config.py b/optimade/validator/config.py index 0c667cb844..194c55c2bc 100644 --- a/optimade/validator/config.py +++ b/optimade/validator/config.py @@ -7,27 +7,25 @@ """ -from typing import Dict, Any, Set, List, Container +from typing import Any, Container, Dict, List, Set + from pydantic import BaseSettings, Field from optimade.models import ( - InfoResponse, - IndexInfoResponse, DataType, + IndexInfoResponse, + InfoResponse, StructureFeatures, SupportLevel, ) +from optimade.server.mappers import BaseResourceMapper +from optimade.server.schemas import ENTRY_INFO_SCHEMAS, retrieve_queryable_properties from optimade.validator.utils import ( ValidatorLinksResponse, - ValidatorReferenceResponseOne, ValidatorReferenceResponseMany, - ValidatorStructureResponseOne, + ValidatorReferenceResponseOne, ValidatorStructureResponseMany, -) -from optimade.server.mappers import BaseResourceMapper -from optimade.server.schemas import ( - ENTRY_INFO_SCHEMAS, - retrieve_queryable_properties, + ValidatorStructureResponseOne, ) __all__ = ("ValidatorConfig", "VALIDATOR_CONFIG") diff --git a/optimade/validator/utils.py b/optimade/validator/utils.py index 1135799185..0c97ce6019 100644 --- a/optimade/validator/utils.py +++ b/optimade/validator/utils.py @@ -12,30 +12,26 @@ """ -import time -import sys -import urllib.parse import dataclasses +import json +import sys +import time import traceback as tb -from typing import List, Optional, Dict, Any, Callable, Tuple - -try: - import simplejson as json -except ImportError: - import json +import urllib.parse +from typing import Any, Callable, Dict, List, Optional, Tuple import requests from pydantic import Field, ValidationError from optimade import __version__ -from optimade.models.optimade_json import Success from optimade.models import ( - ResponseMeta, EntryResource, LinksResource, ReferenceResource, + ResponseMeta, StructureResource, ) +from optimade.models.optimade_json import Success # Default connection timeout allows for one default-sized TCP retransmission window # (see https://docs.python-requests.org/en/latest/user/advanced/#timeouts) @@ -55,22 +51,22 @@ class InternalError(Exception): """ -def print_warning(string, **kwargs): +def print_warning(string: str, **kwargs) -> None: """Print but angry.""" print(f"\033[93m{string}\033[0m", **kwargs) -def print_notify(string, **kwargs): +def print_notify(string: str, **kwargs) -> None: """Print but louder.""" print(f"\033[94m\033[1m{string}\033[0m", **kwargs) -def print_failure(string, **kwargs): +def print_failure(string: str, **kwargs) -> None: """Print but sad.""" print(f"\033[91m\033[1m{string}\033[0m", **kwargs) -def print_success(string, **kwargs): +def print_success(string: str, **kwargs) -> None: """Print but happy.""" print(f"\033[92m\033[1m{string}\033[0m", **kwargs) @@ -117,9 +113,9 @@ def add_success(self, summary: str, success_type: Optional[str] = None): pretty_print = print if success_type == "optional" else print_success if self.verbosity > 0: - pretty_print(message) + pretty_print(message) # type: ignore[operator] elif self.verbosity == 0: - pretty_print(".", end="", flush=True) + pretty_print(".", end="", flush=True) # type: ignore[operator] def add_failure( self, summary: str, message: str, failure_type: Optional[str] = None @@ -150,12 +146,12 @@ def add_failure( self.optional_failure_count += 1 self.optional_failure_messages.append((summary, message)) - pprint_types = { + pprint_types: Dict[str, Tuple[Callable, Callable]] = { "internal": (print_notify, print_warning), "optional": (print, print), } pprint, warning_pprint = pprint_types.get( - failure_type, (print_failure, print_warning) + str(failure_type), (print_failure, print_warning) ) symbol = "!" if failure_type == "internal" else "✖" @@ -172,7 +168,7 @@ def __init__( self, base_url: str, max_retries: int = 5, - headers: Dict[str, str] = None, + headers: Optional[Dict[str, str]] = None, timeout: Optional[float] = DEFAULT_CONN_TIMEOUT, read_timeout: Optional[float] = DEFAULT_READ_TIMEOUT, ) -> None: @@ -196,9 +192,9 @@ def __init__( read_timeout: Read timeout in seconds. """ - self.base_url = base_url - self.last_request = None - self.response = None + self.base_url: str = base_url + self.last_request: Optional[str] = None + self.response: Optional[requests.Response] = None self.max_retries = max_retries self.headers = headers or {} if "User-Agent" not in self.headers: @@ -271,7 +267,7 @@ def get(self, request: str): raise ResponseError(message) -def test_case(test_fn: Callable[[Any], Tuple[Any, str]]): +def test_case(test_fn: Callable[..., Tuple[Any, str]]): """Wrapper for test case functions, which pretty-prints any errors depending on verbosity level, collates the number and severity of test failures, returns the response and summary string to the caller. @@ -294,7 +290,7 @@ def test_case(test_fn: Callable[[Any], Tuple[Any, str]]): def wrapper( validator, *args, - request: str = None, + request: Optional[str] = None, optional: bool = False, multistage: bool = False, **kwargs, @@ -365,7 +361,7 @@ def wrapper( success_type = "optional" if optional else None validator.results.add_success(f"{request} - {msg}", success_type) else: - request = request.replace("\n", "") + request = request.replace("\n", "") # type: ignore[union-attr] message = msg.split("\n") if validator.verbosity > 1: # ValidationErrors from pydantic already include very detailed errors @@ -373,8 +369,7 @@ def wrapper( if not isinstance(result, ValidationError): message += traceback.split("\n") - message = "\n".join(message) - + failure_type = None if isinstance(result, InternalError): summary = ( f"{request} - {test_fn.__name__} - failed with internal error" @@ -385,7 +380,7 @@ def wrapper( failure_type = "optional" if optional else None validator.results.add_failure( - summary, message, failure_type=failure_type + summary, "\n".join(message), failure_type=failure_type ) # set failure result to None as this is expected by other functions @@ -413,13 +408,13 @@ class ValidatorLinksResponse(Success): class ValidatorEntryResponseOne(Success): meta: ResponseMeta = Field(...) data: EntryResource = Field(...) - included: Optional[List[Dict[str, Any]]] = Field(None) + included: Optional[List[Dict[str, Any]]] = Field(None) # type: ignore[assignment] class ValidatorEntryResponseMany(Success): meta: ResponseMeta = Field(...) data: List[EntryResource] = Field(...) - included: Optional[List[Dict[str, Any]]] = Field(None) + included: Optional[List[Dict[str, Any]]] = Field(None) # type: ignore[assignment] class ValidatorReferenceResponseOne(ValidatorEntryResponseOne): diff --git a/optimade/validator/validator.py b/optimade/validator/validator.py index 5f7fbe2445..d60bd2847d 100644 --- a/optimade/validator/validator.py +++ b/optimade/validator/validator.py @@ -6,39 +6,34 @@ class that can be pointed at an OPTIMADE implementation and validated """ # pylint: disable=import-outside-toplevel -import re -import sys +import dataclasses +import json import logging import random +import re +import sys import urllib.parse -import dataclasses -from typing import Union, Tuple, Any, List, Dict, Optional, Set - -try: - import simplejson as json -except ImportError: - import json +from typing import Any, Dict, List, Optional, Set, Tuple, Union import requests from optimade.models import DataType, EntryInfoResponse, SupportLevel +from optimade.validator.config import VALIDATOR_CONFIG as CONF from optimade.validator.utils import ( DEFAULT_CONN_TIMEOUT, DEFAULT_READ_TIMEOUT, Client, - test_case, + ResponseError, + ValidatorEntryResponseMany, + ValidatorEntryResponseOne, + ValidatorResults, print_failure, print_notify, print_success, print_warning, - ResponseError, - ValidatorEntryResponseOne, - ValidatorEntryResponseMany, - ValidatorResults, + test_case, ) -from optimade.validator.config import VALIDATOR_CONFIG as CONF - VERSIONS_REGEXP = r".*/v[0-9]+(\.[0-9]+){,2}$" __all__ = ("ImplementationValidator",) @@ -66,18 +61,18 @@ class ImplementationValidator: def __init__( # pylint: disable=too-many-arguments self, - client: Any = None, - base_url: str = None, + client: Optional[Any] = None, + base_url: Optional[str] = None, verbosity: int = 0, respond_json: bool = False, page_limit: int = 4, max_retries: int = 5, run_optional_tests: bool = True, fail_fast: bool = False, - as_type: str = None, + as_type: Optional[str] = None, index: bool = False, minimal: bool = False, - http_headers: Dict[str, str] = None, + http_headers: Optional[Dict[str, str]] = None, timeout: float = DEFAULT_CONN_TIMEOUT, read_timeout: float = DEFAULT_READ_TIMEOUT, ): @@ -153,11 +148,11 @@ def __init__( # pylint: disable=too-many-arguments f"Not using specified request headers {http_headers} with custom client {self.client}." ) else: - while base_url.endswith("/"): - base_url = base_url[:-1] + while base_url.endswith("/"): # type: ignore[union-attr] + base_url = base_url[:-1] # type: ignore[index] self.base_url = base_url self.client = Client( - base_url, + self.base_url, # type: ignore[arg-type] max_retries=self.max_retries, headers=http_headers, timeout=timeout, @@ -178,8 +173,8 @@ def __init__( # pylint: disable=too-many-arguments self.valid = None - self._test_id_by_type = {} - self._entry_info_by_type = {} + self._test_id_by_type: Dict[str, Any] = {} + self._entry_info_by_type: Dict[str, Any] = {} self.results = ValidatorResults(verbosity=self.verbosity) @@ -355,7 +350,7 @@ def validate_implementation(self): self.print_summary() @test_case - def _recurse_through_endpoint(self, endp: str) -> Tuple[bool, str]: + def _recurse_through_endpoint(self, endp: str) -> Tuple[Optional[bool], str]: """For a given endpoint (`endp`), get the entry type and supported fields, testing that all mandatory fields are supported, then test queries on every property according @@ -452,7 +447,9 @@ def _test_unknown_provider_property(self, endp): "Failed to handle field from unknown provider; should return without affecting filter results" ) - def _check_entry_info(self, entry_info: Dict[str, Any], endp: str) -> List[str]: + def _check_entry_info( + self, entry_info: Dict[str, Any], endp: str + ) -> Dict[str, Dict[str, Any]]: """Checks that `entry_info` contains all the required properties, and returns the property list for the endpoint. @@ -505,7 +502,7 @@ def _test_must_properties( @test_case def _get_archetypal_entry( self, endp: str, properties: List[str] - ) -> Tuple[Dict[str, Any], str]: + ) -> Tuple[Optional[Dict[str, Any]], str]: """Get a random entry from the first page of results for this endpoint. @@ -544,7 +541,9 @@ def _get_archetypal_entry( raise ResponseError(f"Failed to get archetypal entry. Details: {message}") @test_case - def _check_response_fields(self, endp: str, fields: List[str]) -> Tuple[bool, str]: + def _check_response_fields( + self, endp: str, fields: List[str] + ) -> Tuple[Optional[bool], str]: """Check that the response field query parameter is obeyed. Parameters: @@ -1059,7 +1058,7 @@ def _test_multi_entry_endpoint(self, endp: str) -> None: @test_case def _test_data_available_matches_data_returned( self, deserialized: Any - ) -> Tuple[bool, str]: + ) -> Tuple[Optional[bool], str]: """In the case where no query is requested, `data_available` must equal `data_returned` in the meta response, which is tested here. @@ -1159,13 +1158,13 @@ def _test_versions_endpoint_content( f"Version numbers reported by `/{CONF.versions_endpoint}` must be integers specifying the major version, not {text_content}." ) - content_type = response.headers.get("content-type") - if not content_type: + _content_type = response.headers.get("content-type") + if not _content_type: raise ResponseError( "Missing 'Content-Type' in response header from `/versions`." ) - content_type = [_.replace(" ", "") for _ in content_type.split(";")] + content_type = [_.replace(" ", "") for _ in _content_type.split(";")] self._test_versions_headers( content_type, @@ -1234,7 +1233,7 @@ def _test_bad_version_returns_553(self) -> None: error code. """ - expected_status_code = 553 + expected_status_code = [553] if re.match(VERSIONS_REGEXP, self.base_url_parsed.path) is not None: expected_status_code = [404, 400] @@ -1290,12 +1289,12 @@ def _test_page_limit( if previous_links is None: previous_links = set() try: - response = response.json() + response_json = response.json() except (AttributeError, json.JSONDecodeError): raise ResponseError("Unable to test endpoint `page_limit` parameter.") try: - num_entries = len(response["data"]) + num_entries = len(response_json["data"]) except (KeyError, TypeError): raise ResponseError( "Response under `data` field was missing or had wrong type." @@ -1307,13 +1306,13 @@ def _test_page_limit( ) try: - more_data_available = response["meta"]["more_data_available"] + more_data_available = response_json["meta"]["more_data_available"] except KeyError: raise ResponseError("Field `meta->more_data_available` was missing.") if more_data_available and check_next_link: try: - next_link = response["links"]["next"] + next_link = response_json["links"]["next"] if isinstance(next_link, dict): next_link = next_link["href"] except KeyError: @@ -1378,7 +1377,10 @@ def _get_single_id_from_multi_entry_endpoint(self, deserialized): @test_case def _deserialize_response( - self, response: requests.models.Response, response_cls: Any, request: str = None + self, + response: requests.models.Response, + response_cls: Any, + request: Optional[str] = None, ) -> Tuple[Any, str]: """Try to create the appropriate pydantic model from the response. diff --git a/optimade/warnings.py b/optimade/warnings.py new file mode 100644 index 0000000000..4fea44be96 --- /dev/null +++ b/optimade/warnings.py @@ -0,0 +1,83 @@ +"""This submodule implements the possible warnings that can be omitted by an +OPTIMADE API. + +""" + +from typing import Optional + +__all__ = ( + "OptimadeWarning", + "FieldValueNotRecognized", + "TooManyValues", + "QueryParamNotUsed", + "MissingExpectedField", + "TimestampNotRFCCompliant", + "UnknownProviderProperty", + "UnknownProviderQueryParameter", +) + + +class OptimadeWarning(Warning): + """Base Warning for the `optimade` package""" + + def __init__( + self, detail: Optional[str] = None, title: Optional[str] = None, *args + ) -> None: + detail = detail if detail else self.__doc__ + super().__init__(detail, *args) + self.detail = detail + self.title = title if title else self.__class__.__name__ + + def __repr__(self) -> str: + attrs = {"detail": self.detail, "title": self.title} + return "<{:s}({:s})>".format( + self.__class__.__name__, + " ".join( + [ + f"{attr}={value!r}" + for attr, value in attrs.items() + if value is not None + ] + ), + ) + + def __str__(self) -> str: + return self.detail if self.detail is not None else "" + + +class FieldValueNotRecognized(OptimadeWarning): + """A field or value used in the request is not recognised by this implementation.""" + + +class TooManyValues(OptimadeWarning): + """A field or query parameter has too many values to be handled by this implementation.""" + + +class QueryParamNotUsed(OptimadeWarning): + """A query parameter is not used in this request.""" + + +class MissingExpectedField(OptimadeWarning): + """A field was provided with a null value when a related field was provided + with a value.""" + + +class TimestampNotRFCCompliant(OptimadeWarning): + """A timestamp has been used in a filter that contains microseconds and is thus not + RFC 3339 compliant. This may cause undefined behaviour in the query results. + + """ + + +class UnknownProviderProperty(OptimadeWarning): + """A provider-specific property has been requested via `response_fields` or as in a `filter` that is not + recognised by this implementation. + + """ + + +class UnknownProviderQueryParameter(OptimadeWarning): + """A provider-specific query parameter has been requested in the query with a prefix not + recognised by this implementation. + + """ diff --git a/requirements-client.txt b/requirements-client.txt index 3078823557..f80577f739 100644 --- a/requirements-client.txt +++ b/requirements-client.txt @@ -1,5 +1,5 @@ -aiida-core==2.1.1 +aiida-core==2.1.2 ase==3.22.1 jarvis-tools==2022.9.16 -numpy==1.23.4 +numpy==1.23.5 pymatgen==2022.7.25 diff --git a/requirements-dev.txt b/requirements-dev.txt index abcfa17ca2..169b1b3cf7 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,14 @@ +black==22.10.0 build==0.9.0 codecov==2.1.12 +flake8==6.0.0 invoke==1.7.3 +isort==5.10.1 jsondiff==2.0.0 +mypy==0.991 pre-commit==2.20.0 -pylint==2.15.5 +pylint==2.15.6 pytest==7.2.0 pytest-cov==4.0.0 pytest-httpx==0.21.2 +types-all==1.0.0 diff --git a/requirements-docs.txt b/requirements-docs.txt index 9fe89b3b8b..269d98af1c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,5 +1,5 @@ mike==1.1.2 mkdocs==1.4.2 mkdocs-awesome-pages-plugin==2.8.0 -mkdocs-material==8.5.9 +mkdocs-material==8.5.10 mkdocstrings[python-legacy]==0.19.0 diff --git a/requirements-http-client.txt b/requirements-http-client.txt index 4ce1491a34..5cab3b65d0 100644 --- a/requirements-http-client.txt +++ b/requirements-http-client.txt @@ -1,3 +1,3 @@ click==8.1.3 -httpx==0.23.0 +httpx==0.23.1 rich==12.6.0 diff --git a/requirements-server.txt b/requirements-server.txt new file mode 100644 index 0000000000..f6f787b7c9 --- /dev/null +++ b/requirements-server.txt @@ -0,0 +1,5 @@ +elasticsearch==7.17.7 +elasticsearch-dsl==7.4.0 +fastapi==0.86.0 +mongomock==4.1.2 +pymongo==4.3.3 diff --git a/requirements.txt b/requirements.txt index 22498120bf..d307bb6401 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,6 @@ -elasticsearch==7.17.7 -elasticsearch-dsl==7.4.0 email_validator==1.3.0 -fastapi==0.86.0 lark==1.1.4 -mongomock==4.1.2 pydantic==1.10.2 -pymongo==4.3.2 pyyaml==5.4 requests==2.28.1 -uvicorn==0.19.0 +uvicorn==0.20.0 diff --git a/setup.cfg b/setup.cfg index 0d57b10767..39aafd5f0b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,12 @@ filterwarnings = ignore:.*has an unrecognised prefix.*: testpaths = tests addopts = -rs + +[mypy] +plugins = pydantic.mypy +ignore_missing_imports = true +follow_imports = skip + +[isort] +profile = black +known_first_party = optimade diff --git a/setup.py b/setup.py index f6a7df4e8f..4e3b522a87 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ mongo_deps = ["pymongo>=3.12.1,<5", "mongomock~=4.1"] server_deps = [ "uvicorn~=0.19", + "fastapi~=0.86", ] + mongo_deps @@ -57,7 +58,16 @@ "pytest-httpx~=0.21", ] + server_deps dev_deps = ( - ["pylint~=2.15", "pre-commit~=2.20", "invoke~=1.7"] + [ + "black~=22.10", + "flake8~=6.0", + "isort~=5.10", + "mypy~=0.991", + "pylint~=2.15", + "pre-commit~=2.20", + "invoke~=1.7", + "types-all==1.0.0", + ] + docs_deps + testing_deps + client_deps @@ -101,7 +111,6 @@ python_requires=">=3.8", install_requires=[ "lark~=1.1", - "fastapi~=0.86.0", "pydantic~=1.10,>=1.10.2", "email_validator~=1.2", "requests~=2.28", diff --git a/tasks.py b/tasks.py index d2049bf5d0..216cdd7531 100644 --- a/tasks.py +++ b/tasks.py @@ -3,21 +3,19 @@ import re import sys from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional, Tuple from invoke import task from jsondiff import diff if TYPE_CHECKING: - from typing import Tuple - from invoke import Context, Result TOP_DIR = Path(__file__).parent.resolve() -def update_file(filename: str, sub_line: "Tuple[str, str]", strip: str = None): +def update_file(filename: str, sub_line: Tuple[str, str], strip: Optional[str] = None): """Utility function for tasks to read, update, and write files""" with open(filename, "r") as handle: lines = [ @@ -233,14 +231,14 @@ def write_file(full_path: Path, content: str): full_path=docs_dir.joinpath(".pages"), content=pages_template.format(name="API Reference"), ) - continue docs_sub_dir = docs_dir.joinpath(relpath) docs_sub_dir.mkdir(exist_ok=True) - write_file( - full_path=docs_sub_dir.joinpath(".pages"), - content=pages_template.format(name=str(relpath).split("/")[-1]), - ) + if str(relpath) != ".": + write_file( + full_path=docs_sub_dir.joinpath(".pages"), + content=pages_template.format(name=str(relpath).split("/")[-1]), + ) # Create markdown files for filename in filenames: @@ -251,6 +249,10 @@ def write_file(full_path: Path, content: str): basename = filename[: -len(".py")] py_path = f"optimade/{relpath}/{basename}".replace("/", ".") + if str(relpath) == ".": + py_path = py_path.replace("...", ".") + print(filename, basename, py_path) + md_filename = filename.replace(".py", ".md") # For models we want to include EVERYTHING, even if it doesn't have a doc-string diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/adapters/references/__init__.py b/tests/adapters/references/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/adapters/structures/test_jarvis.py b/tests/adapters/structures/test_jarvis.py index d5055756c4..5a44820c7d 100644 --- a/tests/adapters/structures/test_jarvis.py +++ b/tests/adapters/structures/test_jarvis.py @@ -12,6 +12,7 @@ ) from jarvis.core.atoms import Atoms + from optimade.adapters import Structure from optimade.adapters.exceptions import ConversionError from optimade.adapters.structures.jarvis import get_jarvis_atoms diff --git a/tests/adapters/structures/test_pymatgen.py b/tests/adapters/structures/test_pymatgen.py index 7fda95d564..b27de65c92 100644 --- a/tests/adapters/structures/test_pymatgen.py +++ b/tests/adapters/structures/test_pymatgen.py @@ -11,14 +11,15 @@ " be able to run", ) -from pymatgen.core import Molecule, Structure as PymatgenStructure +from pymatgen.core import Molecule +from pymatgen.core import Structure as PymatgenStructure from optimade.adapters import Structure from optimade.adapters.structures.pymatgen import ( - get_pymatgen, - _get_structure, _get_molecule, + _get_structure, from_pymatgen, + get_pymatgen, ) diff --git a/tests/adapters/structures/test_structures.py b/tests/adapters/structures/test_structures.py index f7419c8fdd..fd82033cfc 100644 --- a/tests/adapters/structures/test_structures.py +++ b/tests/adapters/structures/test_structures.py @@ -1,5 +1,6 @@ """Test Structure adapter""" import pytest + from optimade.adapters import Structure from optimade.models import StructureResource diff --git a/tests/adapters/structures/test_utils.py b/tests/adapters/structures/test_utils.py index 4dd18a9cee..5cb1e76340 100644 --- a/tests/adapters/structures/test_utils.py +++ b/tests/adapters/structures/test_utils.py @@ -19,7 +19,6 @@ species_from_species_at_sites, ) - # TODO: Add tests for cell_to_cellpar, unit_vector, cellpar_to_cell diff --git a/tests/adapters/structures/utils.py b/tests/adapters/structures/utils.py index 8883a1d7df..2129ad6aa6 100644 --- a/tests/adapters/structures/utils.py +++ b/tests/adapters/structures/utils.py @@ -1,5 +1,5 @@ -from pathlib import Path import re +from pathlib import Path def get_min_ver(dependency: str) -> str: diff --git a/tests/conftest.py b/tests/conftest.py index 3ea1cad6aa..266f5adb7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def mapper(): """Mapper-factory to import a mapper from optimade.server.mappers""" from optimade.server import mappers - def _mapper(name: str) -> mappers.BaseResourceMapper: + def _mapper(name: str) -> mappers.BaseResourceMapper: # type: ignore[return] """Return named resource mapper""" try: res = getattr(mappers, name) diff --git a/tests/filterparser/test_filterparser.py b/tests/filterparser/test_filterparser.py index 99007b9dc3..5296bf797e 100644 --- a/tests/filterparser/test_filterparser.py +++ b/tests/filterparser/test_filterparser.py @@ -2,11 +2,10 @@ from typing import Tuple import pytest - from lark import Tree +from optimade.exceptions import BadRequest from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest class BaseTestFilterParser(abc.ABC): diff --git a/tests/filtertransformers/test_elasticsearch.py b/tests/filtertransformers/test_elasticsearch.py index da70d95e6c..3d076a7915 100644 --- a/tests/filtertransformers/test_elasticsearch.py +++ b/tests/filtertransformers/test_elasticsearch.py @@ -1,7 +1,8 @@ import pytest elasticsearch_dsl = pytest.importorskip( - "elasticsearch_dsl", reason="No ElasticSearch installation, skipping tests..." + "elasticsearch_dsl", + reason="ElasticSearch dependencies (elasticsearch_dsl, elasticsearch) are required to run these tests.", ) from optimade.filterparser import LarkParser diff --git a/tests/filtertransformers/test_mongo.py b/tests/filtertransformers/test_mongo.py index 22d243a26f..b4dd9cced9 100644 --- a/tests/filtertransformers/test_mongo.py +++ b/tests/filtertransformers/test_mongo.py @@ -1,10 +1,15 @@ import pytest +_ = pytest.importorskip( + "bson", + reason="MongoDB dependency set (pymongo, bson) are required to run these tests.", +) + from lark.exceptions import VisitError +from optimade.exceptions import BadRequest from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest -from optimade.server.warnings import UnknownProviderProperty +from optimade.warnings import UnknownProviderProperty class TestMongoTransformer: @@ -523,9 +528,11 @@ def test_list_length_aliases(self, mapper): def test_suspected_timestamp_fields(self, mapper): import datetime + import bson.tz_util + from optimade.filtertransformers.mongo import MongoTransformer - from optimade.server.warnings import TimestampNotRFCCompliant + from optimade.warnings import TimestampNotRFCCompliant example_RFC3339_date = "2019-06-08T04:13:37Z" example_RFC3339_date_2 = "2019-06-08T04:13:37" @@ -589,9 +596,10 @@ def test_unaliased_length_operator(self): def test_mongo_special_id(self, mapper): - from optimade.filtertransformers.mongo import MongoTransformer from bson import ObjectId + from optimade.filtertransformers.mongo import MongoTransformer + class MyMapper(mapper("StructureMapper")): ALIASES = (("immutable_id", "_id"),) diff --git a/tests/models/conftest.py b/tests/models/conftest.py index ff0daebee1..a6cc485646 100644 --- a/tests/models/conftest.py +++ b/tests/models/conftest.py @@ -49,6 +49,15 @@ def good_structures() -> list: return structures +@pytest.fixture(scope="session") +def good_references() -> list: + """Load and return list of good structures resources""" + filename = "test_good_references.json" + references = load_test_data(filename) + references = remove_mongo_date(references) + return references + + @pytest.fixture def starting_links() -> dict: """A good starting links resource""" diff --git a/tests/models/test_data/test_good_references.json b/tests/models/test_data/test_good_references.json new file mode 100644 index 0000000000..bf55373be0 --- /dev/null +++ b/tests/models/test_data/test_good_references.json @@ -0,0 +1,50 @@ +[ + { + "id": "dijkstra1968", + "type": "references", + "last_modified": "2019-11-12T14:24:37.331000", + "authors": [ + { + "name": "Edsger W. Dijkstra", + "firstname": "Edsger", + "lastname": "Dijkstra" + } + ], + "doi": "10.1145/362929.362947", + "journal": "Communications of the ACM", + "title": "Go To Statement Considered Harmful", + "year": "1968" + }, + { + "id": "maddox1988", + "type": "references", + "last_modified": "2019-11-27T14:24:37.331000", + "authors": [ + { + "name": "John Maddox", + "firstname": "John", + "lastname": "Maddox" + } + ], + "doi": "10.1038/335201a0", + "journal": "Nature", + "title": "Crystals From First Principles", + "year": "1988" + }, + { + "id": "dummy/2019", + "type": "references", + "last_modified": "2019-11-23T14:24:37.332000", + "authors": [ + { + "name": "A Nother", + "firstname": "A", + "lastname": "Nother" + } + ], + "doi": "10.1038/00000", + "journal": "JACS", + "title": "Dummy reference that should remain orphaned from all structures for testing purposes", + "year": "2019" + } +] diff --git a/tests/models/test_entries.py b/tests/models/test_entries.py index ff5f28b1e4..a3ab7e318a 100644 --- a/tests/models/test_entries.py +++ b/tests/models/test_entries.py @@ -1,5 +1,4 @@ import pytest - from pydantic import ValidationError from optimade.models.entries import EntryRelationships diff --git a/tests/models/test_jsonapi.py b/tests/models/test_jsonapi.py index 9cb66d2e0b..b2e09cb4a6 100644 --- a/tests/models/test_jsonapi.py +++ b/tests/models/test_jsonapi.py @@ -1,5 +1,5 @@ -from pydantic import ValidationError import pytest +from pydantic import ValidationError def test_hashability(): diff --git a/tests/models/test_links.py b/tests/models/test_links.py index 838bb813b0..c65cc9edf5 100644 --- a/tests/models/test_links.py +++ b/tests/models/test_links.py @@ -3,18 +3,11 @@ from optimade.models.links import LinksResource - MAPPER = "LinksMapper" def test_good_links(starting_links, mapper): """Check well-formed links used as example data""" - import optimade.server.data - - good_refs = optimade.server.data.links - for doc in good_refs: - LinksResource(**mapper(MAPPER).map_back(doc)) - # Test starting_links is a good links resource LinksResource(**mapper(MAPPER).map_back(starting_links)) diff --git a/tests/models/test_references.py b/tests/models/test_references.py index be8a9522d1..de86bebb48 100644 --- a/tests/models/test_references.py +++ b/tests/models/test_references.py @@ -1,19 +1,23 @@ # pylint: disable=no-member import pytest +from pydantic import ValidationError from optimade.models.references import ReferenceResource - MAPPER = "ReferenceMapper" -def test_good_references(mapper): - """Check well-formed references used as example data""" - import optimade.server.data - - good_refs = optimade.server.data.references - for doc in good_refs: - ReferenceResource(**mapper(MAPPER).map_back(doc)) +def test_more_good_references(good_references, mapper): + """Check well-formed structures with specific edge-cases""" + for index, structure in enumerate(good_references): + try: + ReferenceResource(**mapper(MAPPER).map_back(structure)) + except ValidationError: + # Printing to keep the original exception as is, while still being informational + print( + f"Good test structure {index} failed to validate from 'test_good_structures.json'" + ) + raise def test_bad_references(mapper): diff --git a/tests/models/test_structures.py b/tests/models/test_structures.py index d31572d532..60f8115636 100644 --- a/tests/models/test_structures.py +++ b/tests/models/test_structures.py @@ -2,21 +2,12 @@ import itertools import pytest -from optimade.models.structures import CORRELATED_STRUCTURE_FIELDS, StructureResource -from optimade.server.warnings import MissingExpectedField from pydantic import ValidationError -MAPPER = "StructureMapper" - - -def test_good_structures(mapper): - """Check well-formed structures used as example data""" - import optimade.server.data - - good_structures = optimade.server.data.structures +from optimade.models.structures import CORRELATED_STRUCTURE_FIELDS, StructureResource +from optimade.warnings import MissingExpectedField - for structure in good_structures: - StructureResource(**mapper(MAPPER).map_back(structure)) +MAPPER = "StructureMapper" @pytest.mark.filterwarnings("ignore", category=MissingExpectedField) diff --git a/tests/models/test_utils.py b/tests/models/test_utils.py index adf3221671..955ecdee95 100644 --- a/tests/models/test_utils.py +++ b/tests/models/test_utils.py @@ -1,7 +1,9 @@ +from typing import Callable, List + import pytest -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError + from optimade.models.utils import OptimadeField, StrictField, SupportLevel -from typing import List, Callable def make_bad_models(field: Callable): @@ -82,6 +84,7 @@ def test_formula_regexp(): """ import re + from optimade.models.utils import CHEMICAL_FORMULA_REGEXP class DummyModel(BaseModel): diff --git a/tests/server/conftest.py b/tests/server/conftest.py index 3f071e1ba5..175d3f8fa0 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -1,9 +1,9 @@ -from typing import Union, Dict -from optimade.server.warnings import OptimadeWarning - +from typing import Dict, Optional, Union import pytest +from optimade.warnings import OptimadeWarning + @pytest.fixture(scope="session") def client(): @@ -54,10 +54,7 @@ def both_fake_remote_clients(request): @pytest.fixture def get_good_response(client, index_client): """Get response with some sanity checks, expecting '200 OK'""" - try: - import simplejson as json - except ImportError: - import json + import json from requests import Response @@ -128,16 +125,18 @@ def check_response(get_good_response): """ from typing import List + from optimade.server.config import CONFIG + from .utils import OptimadeTestClient def inner( request: str, expected_ids: Union[str, List[str]], page_limit: int = CONFIG.page_limit, - expected_return: int = None, + expected_return: Optional[int] = None, expected_as_is: bool = False, - expected_warnings: List[Dict[str, str]] = None, + expected_warnings: Optional[List[Dict[str, str]]] = None, server: Union[str, OptimadeTestClient] = "regular", ): if expected_warnings: @@ -182,9 +181,9 @@ def check_error_response(client, index_client): def inner( request: str, - expected_status: int = None, - expected_title: str = None, - expected_detail: str = None, + expected_status: Optional[int] = None, + expected_title: Optional[str] = None, + expected_detail: Optional[str] = None, server: Union[str, OptimadeTestClient] = "regular", ): response = None diff --git a/tests/server/middleware/test_api_hint.py b/tests/server/middleware/test_api_hint.py index ebee0ecbf8..da4aa3b7d6 100644 --- a/tests/server/middleware/test_api_hint.py +++ b/tests/server/middleware/test_api_hint.py @@ -1,7 +1,8 @@ """Test HandleApiHint middleware and the `api_hint` query parameter""" -import pytest from urllib.parse import unquote +import pytest + from optimade.server.exceptions import VersionNotSupported from optimade.server.middleware import HandleApiHint diff --git a/tests/server/middleware/test_query_param.py b/tests/server/middleware/test_query_param.py index 3e4ad133d8..03998a2dcd 100644 --- a/tests/server/middleware/test_query_param.py +++ b/tests/server/middleware/test_query_param.py @@ -1,8 +1,9 @@ """Test EntryListingQueryParams middleware""" import pytest -from optimade.server.exceptions import BadRequest + +from optimade.exceptions import BadRequest from optimade.server.middleware import EnsureQueryParamIntegrity -from optimade.server.warnings import FieldValueNotRecognized +from optimade.warnings import FieldValueNotRecognized def test_wrong_html_form(check_error_response, both_clients): diff --git a/tests/server/query_params/__init__.py b/tests/server/query_params/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/server/query_params/conftest.py b/tests/server/query_params/conftest.py index bae882886c..ce11819be2 100644 --- a/tests/server/query_params/conftest.py +++ b/tests/server/query_params/conftest.py @@ -12,13 +12,13 @@ def structures(): @pytest.fixture def check_include_response(get_good_response): """Fixture to check "good" `include` response""" - from typing import Union, List, Set + from typing import List, Optional, Set, Union def inner( request: str, expected_included_types: Union[List, Set], expected_included_resources: Union[List, Set], - expected_relationship_types: Union[List, Set] = None, + expected_relationship_types: Optional[Union[List, Set]] = None, server: str = "regular", ): response = get_good_response(request, server) diff --git a/tests/server/query_params/test_filter.py b/tests/server/query_params/test_filter.py index b141906cfd..abc6fb94dc 100644 --- a/tests/server/query_params/test_filter.py +++ b/tests/server/query_params/test_filter.py @@ -1,5 +1,6 @@ """Make sure filters are handled correctly""" import pytest + from optimade.server.config import CONFIG, SupportedBackend diff --git a/tests/server/routers/test_info.py b/tests/server/routers/test_info.py index dbb0419b8d..2fd27e0ee2 100644 --- a/tests/server/routers/test_info.py +++ b/tests/server/routers/test_info.py @@ -1,6 +1,6 @@ -from optimade.models import InfoResponse, EntryInfoResponse, IndexInfoResponse, DataType +from optimade.models import DataType, EntryInfoResponse, IndexInfoResponse, InfoResponse -from ..utils import RegularEndpointTests, IndexEndpointTests +from ..utils import IndexEndpointTests, RegularEndpointTests class TestInfoEndpoint(RegularEndpointTests): diff --git a/tests/server/routers/test_structures.py b/tests/server/routers/test_structures.py index eefdfe4a6b..a70d7ebc81 100644 --- a/tests/server/routers/test_structures.py +++ b/tests/server/routers/test_structures.py @@ -1,7 +1,7 @@ from optimade.models import ( + ReferenceResource, StructureResponseMany, StructureResponseOne, - ReferenceResource, ) from ..utils import RegularEndpointTests diff --git a/tests/server/routers/test_utils.py b/tests/server/routers/test_utils.py index 4d0ee11378..37b1f8ee55 100644 --- a/tests/server/routers/test_utils.py +++ b/tests/server/routers/test_utils.py @@ -2,9 +2,8 @@ from typing import Mapping, Optional, Tuple, Union from unittest import mock -from requests.exceptions import ConnectionError - import pytest +from requests.exceptions import ConnectionError def mocked_providers_list_response( @@ -20,7 +19,7 @@ def mocked_providers_list_response( https://stackoverflow.com/questions/15753390/how-can-i-mock-requests-and-the-response """ try: - from optimade.server.data import providers + from optimade.server.data import providers # type: ignore[attr-defined] except ImportError: pytest.fail( "Cannot import providers from optimade.server.data, " @@ -76,7 +75,8 @@ def test_get_providers(): def test_get_providers_warning(caplog, top_dir): """Make sure a warning is logged as a last resort.""" import copy - from optimade.server.routers.utils import get_providers, PROVIDER_LIST_URLS + + from optimade.server.routers.utils import PROVIDER_LIST_URLS, get_providers providers_cache = False try: diff --git a/tests/server/routers/test_versions.py b/tests/server/routers/test_versions.py index f2dd9ba352..be61af6662 100644 --- a/tests/server/routers/test_versions.py +++ b/tests/server/routers/test_versions.py @@ -1,4 +1,5 @@ from optimade import __api_version__ + from ..utils import NoJsonEndpointTests diff --git a/tests/server/test_client.py b/tests/server/test_client.py index 3aa86f4308..89846e02f0 100644 --- a/tests/server/test_client.py +++ b/tests/server/test_client.py @@ -2,9 +2,9 @@ from functools import partial from pathlib import Path - import pytest -from optimade.server.warnings import MissingExpectedField + +from optimade.warnings import MissingExpectedField try: from optimade.client import OptimadeClient @@ -37,34 +37,42 @@ def test_client_endpoints(httpx_mocked_response, use_async): filter = "" cli = OptimadeClient(base_urls=[TEST_URL], use_async=use_async) - results = cli.get() - assert results["structures"][filter][TEST_URL]["data"] - assert results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + get_results = cli.get() + assert get_results["structures"][filter][TEST_URL]["data"] + assert ( + get_results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + ) - results = cli.structures.get() - assert results["structures"][filter][TEST_URL]["data"] - assert results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + get_results = cli.structures.get() + assert get_results["structures"][filter][TEST_URL]["data"] + assert ( + get_results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + ) - results = cli.references.get() - assert results["references"][filter][TEST_URL]["data"] - assert results["references"][filter][TEST_URL]["data"][0]["type"] == "references" + get_results = cli.references.get() + assert get_results["references"][filter][TEST_URL]["data"] + assert ( + get_results["references"][filter][TEST_URL]["data"][0]["type"] == "references" + ) - results = cli.get() - assert results["structures"][filter][TEST_URL]["data"] - assert results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + get_results = cli.get() + assert get_results["structures"][filter][TEST_URL]["data"] + assert ( + get_results["structures"][filter][TEST_URL]["data"][0]["type"] == "structures" + ) - results = cli.references.count() - assert results["references"][filter][TEST_URL] > 0 + count_results = cli.references.count() + assert count_results["references"][filter][TEST_URL] > 0 filter = 'elements HAS "Ag"' - results = cli.count(filter) - assert results["structures"][filter][TEST_URL] > 0 + count_results = cli.count(filter) + assert count_results["structures"][filter][TEST_URL] > 0 - results = cli.info.get() - assert results["info"][""][TEST_URL]["data"]["type"] == "info" + count_results = cli.info.get() + assert count_results["info"][""][TEST_URL]["data"]["type"] == "info" - results = cli.info.structures.get() - assert "properties" in results["info/structures"][""][TEST_URL]["data"] + count_results = cli.info.structures.get() + assert "properties" in count_results["info/structures"][""][TEST_URL]["data"] @pytest.mark.parametrize("use_async", [False]) diff --git a/tests/server/test_config.py b/tests/server/test_config.py index 86990f1fbd..db0175b3f5 100644 --- a/tests/server/test_config.py +++ b/tests/server/test_config.py @@ -1,7 +1,6 @@ # pylint: disable=protected-access,pointless-statement,relative-beyond-top-level import json import os - from pathlib import Path @@ -141,7 +140,9 @@ def test_yaml_config_file(): NOTE: Since pytest loads a JSON config file, there's no need to test that further. """ import tempfile + import pytest + from optimade.server.config import ServerConfig yaml_content = """ diff --git a/tests/server/test_mappers.py b/tests/server/test_mappers.py index 9876d5718a..4b6899d8a0 100644 --- a/tests/server/test_mappers.py +++ b/tests/server/test_mappers.py @@ -1,8 +1,7 @@ import pytest -from optimade.server.config import CONFIG from optimade.models import StructureResource - +from optimade.server.config import CONFIG MAPPER = "BaseResourceMapper" diff --git a/tests/server/test_server_validation.py b/tests/server/test_server_validation.py index 2660121939..2d2a38f7c5 100644 --- a/tests/server/test_server_validation.py +++ b/tests/server/test_server_validation.py @@ -1,5 +1,5 @@ -import json import dataclasses +import json import pytest @@ -44,7 +44,7 @@ def test_with_validator_json_response(both_fake_remote_clients, capsys): def test_as_type_with_validator(client, capsys): - from unittest.mock import patch, Mock + from unittest.mock import Mock, patch test_urls = { f"{client.base_url}/structures": "structures", @@ -95,10 +95,6 @@ def test_versioned_base_urls(client, index_client, server: str): This depends on the routers for each kind of server. """ - try: - import simplejson as json - except ImportError: - import json from optimade.server.routers.utils import BASE_URL_PREFIXES @@ -132,13 +128,9 @@ def test_meta_schema_value_obeys_index(client, index_client, server: str): """Test that the reported `meta->schema` is correct for index/non-index servers. """ - try: - import simplejson as json - except ImportError: - import json - from optimade.server.routers.utils import BASE_URL_PREFIXES from optimade.server.config import CONFIG + from optimade.server.routers.utils import BASE_URL_PREFIXES clients = { "regular": client, diff --git a/tests/server/utils.py b/tests/server/utils.py index e45898cd7e..c92075be90 100644 --- a/tests/server/utils.py +++ b/tests/server/utils.py @@ -1,20 +1,15 @@ +import json import re -import typing -from urllib.parse import urlparse import warnings - -try: - import simplejson as json -except ImportError: - import json +from typing import Iterable, Optional, Type, Union +from urllib.parse import urlparse import pytest -from requests import Response - -from pydantic import BaseModel # pylint: disable=no-name-in-module from fastapi.testclient import TestClient +from requests import Response from starlette import testclient +import optimade.models.jsonapi as jsonapi from optimade import __api_version__ from optimade.models import ResponseMeta @@ -29,7 +24,7 @@ class OptimadeTestClient(TestClient): def __init__( self, - app: typing.Union[testclient.ASGI2App, testclient.ASGI3App], + app: Union[testclient.ASGI2App, testclient.ASGI3App], base_url: str = "http://example.org", raise_server_exceptions: bool = True, root_path: str = "", @@ -58,20 +53,7 @@ def request( # pylint: disable=too-many-locals self, method: str, url: str, - params: testclient.Params = None, - data: testclient.DataType = None, - headers: typing.MutableMapping[str, str] = None, - cookies: testclient.Cookies = None, - files: testclient.FileType = None, - auth: testclient.AuthType = None, - timeout: testclient.TimeOut = None, - allow_redirects: bool = None, - proxies: typing.MutableMapping[str, str] = None, - hooks: typing.Any = None, - stream: bool = None, - verify: typing.Union[bool, str] = None, - cert: typing.Union[str, typing.Tuple[str, str]] = None, - json: typing.Any = None, # pylint: disable=redefined-outer-name + **kwargs, ) -> Response: if ( re.match(r"/?v[0-9](.[0-9]){0,2}/", url) is None @@ -83,34 +65,21 @@ def request( # pylint: disable=too-many-locals return super(OptimadeTestClient, self).request( method=method, url=url, - params=params, - data=data, - headers=headers, - cookies=cookies, - files=files, - auth=auth, - timeout=timeout, - allow_redirects=allow_redirects, - proxies=proxies, - hooks=hooks, - stream=stream, - verify=verify, - cert=cert, - json=json, + **kwargs, ) class BaseEndpointTests: """Base class for common tests of endpoints""" - request_str: str = None - response_cls: BaseModel = None + request_str: Optional[str] = None + response_cls: Optional[Type[jsonapi.Response]] = None - response: Response = None - json_response: dict = None + response: Optional[Response] = None + json_response: Optional[dict] = None @staticmethod - def check_keys(keys: list, response_subset: typing.Iterable): + def check_keys(keys: list, response_subset: Iterable): for key in keys: assert ( key in response_subset @@ -193,7 +162,7 @@ def client_factory(): """Return TestClient for OPTIMADE server""" def inner( - version: str = None, + version: Optional[str] = None, server: str = "regular", raise_server_exceptions: bool = True, add_empty_endpoint: bool = False, @@ -209,35 +178,26 @@ def inner( responses (`add_empty_endpoint`) """ - if server == "regular": - from optimade.server.main import ( - app, - add_major_version_base_url, - add_optional_versioned_base_urls, - ) - elif server == "index": - from optimade.server.main_index import ( - app, - add_major_version_base_url, - add_optional_versioned_base_urls, - ) - else: - pytest.fail( - f"Wrong value for 'server': {server}. It must be either 'regular' or 'index'." - ) + import importlib - add_major_version_base_url(app) - add_optional_versioned_base_urls(app) + module_name = "optimade.server.main" + if server == "index": + module_name += "_index" + server_module = importlib.import_module(module_name) + app = server_module.app + server_module.add_major_version_base_url(app) + server_module.add_optional_versioned_base_urls(app) if add_empty_endpoint: - from starlette.routing import Router, Route + from fastapi import APIRouter from fastapi.responses import PlainTextResponse + from starlette.routing import Route async def empty(_): return PlainTextResponse(b"", 200) - empty_router = Router( + empty_router = APIRouter( routes=[Route("/extensions/test_empty_body", endpoint=empty)] ) app.include_router(empty_router) @@ -261,10 +221,10 @@ async def empty(_): class NoJsonEndpointTests: """A simplified mixin class for tests on non-JSON endpoints.""" - request_str: str = None - response_cls: BaseModel = None + request_str: Optional[str] = None + response_cls: Optional[Type] = None - response: Response = None + response: Optional[Response] = None @pytest.fixture(autouse=True) def get_response(self, both_clients): diff --git a/tests/test_setup.py b/tests/test_setup.py index 25370f83fd..b1b731dc8e 100644 --- a/tests/test_setup.py +++ b/tests/test_setup.py @@ -1,12 +1,7 @@ """Distribution tests.""" -from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: - from typing import List - - package_data = [ ".lark", "index_links.json", @@ -38,7 +33,7 @@ def build_dist() -> str: @pytest.mark.parametrize("package_file", package_data) -def test_distribution_package_data(package_file: "List[str]", build_dist: str) -> None: +def test_distribution_package_data(package_file: str, build_dist: str) -> None: """Make sure a distribution has all the needed package data.""" import re diff --git a/tests/validator/test_utils.py b/tests/validator/test_utils.py index 4e34b2db31..5503f3fa88 100644 --- a/tests/validator/test_utils.py +++ b/tests/validator/test_utils.py @@ -1,13 +1,11 @@ +import json + import pytest -from optimade.validator.utils import test_case as validator_test_case + from optimade.validator.utils import ResponseError +from optimade.validator.utils import test_case as validator_test_case from optimade.validator.validator import ImplementationValidator -try: - import simplejson as json -except ImportError: - import json - @validator_test_case def dummy_test_case(_, returns, raise_exception=None): @@ -371,9 +369,10 @@ def test_that_system_exit_is_fatal_in_test_case(): def test_versions_test_cases(): """Check the `/versions` test cases.""" - from requests import Response from functools import partial + from requests import Response + unversioned_base_url = "https://example.org" versioned_base_url = unversioned_base_url + "/v1"