Merged master into allow nested_aliases branch.

.github/fly.toml

@@ -0,0 +1,38 @@
+# fly.toml file generated for broken-field-1976 on 2022-11-21T21:37:55Z
+
+app = "optimade"
+kill_signal = "SIGINT"
+kill_timeout = 5
+processes = []
+
+[env]
+
+[experimental]
+  allowed_public_ports = []
+  auto_rollback = true
+
+[[services]]
+  http_checks = []
+  internal_port = 5000
+  processes = ["app"]
+  protocol = "tcp"
+  script_checks = []
+  [services.concurrency]
+    hard_limit = 25
+    soft_limit = 20
+    type = "connections"
+
+  [[services.ports]]
+    force_https = true
+    handlers = ["http"]
+    port = 80
+
+  [[services.ports]]
+    handlers = ["tls", "http"]
+    port = 443
+
+  [[services.tcp_checks]]
+    grace_period = "1s"
+    interval = "15s"
+    restart_limit = 0
+    timeout = "2s"

.github/workflows/ci.yml

@@ -7,6 +7,9 @@ on:
     - master
     - 'push-action/**'
 
+env:
+  PYTEST_ADDOPTS: "--color=yes"
+
 # Cancel running workflows when additional changes are pushed
 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-a-fallback-value
 concurrency:
@@ -162,7 +165,7 @@ jobs:
         ports:
         - 5432:5432
       elasticsearch:
-        image: elasticsearch:7.17.1
+        image: elasticsearch:7.17.7
         ports:
           - 9200:9200
           - 9300:9300
@@ -246,22 +249,22 @@ jobs:
         coverage run --append --source optimade optimade/client/cli.py \
            --filter 'nsites = 1' \
            --output-file test_get_async.json \
-           https://optimade.herokuapp.com
+           https://optimade.fly.dev
         test test_get_async.json
 
         coverage run --append --source optimade optimade/client/cli.py \
           --filter 'nsites = 1' \
           --count \
           --output-file test_count.json \
-          https://optimade.herokuapp.com
+          https://optimade.fly.dev
         test test_count.json
 
         coverage run --append --source optimade optimade/client/cli.py \
           --no-async \
           --filter 'nsites = 1' \
           --count \
           --output-file test_count_no_async.json \
-          https://optimade.herokuapp.com
+          https://optimade.fly.dev
         test test_count_no_async.json
 
         diff test_count_no_async.json test_count.json

.github/workflows/ci_cd_updated_master.yml

@@ -135,23 +135,25 @@ jobs:
         sleep: 15
         force: ${{ env.FORCE_PUSH }}
 
-  deploy-heroku:
+  deploy-fly:
+    env:
+        FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        OPTIMADE_CONFIG_FILE: /app/tests/test_config.json
+        OPTIMADE_BASE_URL: https://optimade.fly.dev
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Materials-Consortia'
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
-
-    - name: Deploy to Heroku
-      uses: akhileshns/[email protected]
       with:
-          heroku_api_key: ${{secrets.HEROKU_API_KEY}}
-          heroku_app_name: "optimade"
-          heroku_email: ${{secrets.HEROKU_EMAIL}}
-          branch: ${{env.DEFAULT_REPO_BRANCH}}
-      env:
-          HD_OPTIMADE_CONFIG_FILE: /app/tests/test_config.json
-          HD_OPTIMADE_BASE_URL: https://optimade.herokuapp.com
+        fetch-depth: 0
+        submodules: true
+
+    - name: Set up Fly
+      uses: superfly/flyctl-actions/setup-flyctl@master
+
+    - name: Deploy to Fly
+      run: flyctl deploy --remote-only -c ./.github/fly.toml
 
   publish_container_image:
     name: Publish container image

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
       args: [--markdown-linebreak-ext=md]
 
   - repo: https://github.com/pycqa/flake8
-    rev: '3.9.2'
+    rev: '5.0.4'
     hooks:
     - id: flake8
 

INSTALL.md

@@ -101,7 +101,7 @@ These commands should be run from a local optimade-python-tools directory.
 The following command starts a local Elasticsearch v6 instance, runs the test suite, then stops and deletes the containers (required as the tests insert some data):
 
 ```shell
-docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:7.17.1 \
+docker run -d --name elasticsearch_test -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "xpack.security.enabled=false" elasticsearch:7.17.7 \
 && sleep 10 \
 && OPTIMADE_DATABASE_BACKEND="elastic" py.test; \
 docker container stop elasticsearch_test; docker container rm elasticsearch_test

README.md

@@ -31,7 +31,6 @@ OPTIMADE Python tools
   <td align="center">
     <a href="https://github.com/Materials-Consortia/optimade-python-tools/actions?query=branch%3Amaster+"><img alt="Build Status" src="https://img.shields.io/github/workflow/status/Materials-Consortia/optimade-python-tools/CI%20tests?logo=github"></a><br>
     <a href="https://codecov.io/gh/Materials-Consortia/optimade-python-tools"><img alt="Codecov" src="https://img.shields.io/codecov/c/github/Materials-Consortia/optimade-python-tools?logo=codecov&logoColor=white&token=UJAtmqkZZO"></a><br>
-    <a href="https://optimade.herokuapp.com"><img alt="Heroku App Status"  src="https://heroku-shields.herokuapp.com/optimade??logo=heroku"></a>
   </td>
   <td align="center">
     <a href="https://github.com/Materials-Consortia/optimade-python-tools/pulse"><img alt="Commit Activity" src="https://img.shields.io/github/commit-activity/m/Materials-Consortia/optimade-python-tools?logo=github"></a><br>
@@ -50,7 +49,7 @@ This is to enable interoperability among databases that serve crystal structures
 This repository contains a library of tools for implementing and consuming [OPTIMADE APIs](https://www.optimade.org) using Python:
 
 1. [pydantic](https://github.com/pydantic/pydantic) data models for all [OPTIMADE entry types](https://www.optimade.org/optimade-python-tools/latest/all_models/) and endpoint responses, and a [Lark](https://github.com/lark-parser/lark) [EBNF grammar](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) implementation for the OPTIMADE filter language.
-1. A configurable reference server implementation that can make use of either MongoDB or Elasticsearch database backends out-of-the-box, and is readily extensible to other backends. Try it out on the [demo site](https://optimade.herokuapp.com)! The OpenAPI schemas of the server are used to construct the [OPTIMADE schemas](https://schemas.optimade.org/) site.
+1. A configurable reference server implementation that can make use of either MongoDB or Elasticsearch database backends out-of-the-box, and is readily extensible to other backends. Try it out on the [demo site](https://optimade.fly.dev)! The OpenAPI schemas of the server are used to construct the [OPTIMADE schemas](https://schemas.optimade.org/) site.
 1. An [OPTIMADE client](https://www.optimade.org/optimade-python-tools/latest/getting_started/client/) (`optimade-get`) that can query multiple [OPTIMADE providers](https://optimade.org/providers-dashboard) concurrently with a given filter, at the command-line or from Python code.
 1. A fuzzy API validator tool, which may be called from the shell (`optimade-validator`) or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action); this validator is used to construct the [providers dashboard](https://optimade.org/providers-dashboard).
 

docs/concepts/validation.md

@@ -12,8 +12,8 @@ To run the script, simply provide an OPTIMADE URL to the script at the command-l
 You can use the following to validate the Heroku deployment of our reference server:
 
 ```shell
-$ optimade-validator https://optimade.herokuapp.com/
-Testing entire implementation at https://optimade.herokuapp.com
+$ optimade-validator https://optimade.fly.dev/
+Testing entire implementation at https://optimade.fly.dev
 ...
 ```
 

docs/getting_started/client.md

@@ -60,14 +60,14 @@ We can refine the search by manually specifying some URLs:
 
 === "Command line"
     ```shell
-    optimade-get --output-file results.json https://optimade.herokuapp.com https://optimade.odbx.science
+    optimade-get --output-file results.json https://optimade.fly.dev https://optimade.odbx.science
     ```
 
 === "Python"
     ```python
     from optimade.client import OptimadeClient
     client = OptimadeClient(
-        base_urls=["https://optimade.herokuapp.com", "https://optimade.odbx.science"]
+        base_urls=["https://optimade.fly.dev", "https://optimade.odbx.science"]
     )
     client.get()
     ```
@@ -97,22 +97,22 @@ At the command-line, the results of the query will be printed to `stdout`, ready
 For example:
 
 ```shell
-optimade-get --filter 'nsites = 1' --output-file results.json https://optimade.herokuapp.com
+optimade-get --filter 'nsites = 1' --output-file results.json https://optimade.fly.dev
 cat results.json
 ```
 
 has the following (truncated) output:
 
 ```json
-{  
+{
   // The endpoint that was queried
-  "structures": {  
+  "structures": {
     // The filter applied to that endpointk
-    "nsites = 1": {  
+    "nsites = 1": {
       // The base URL of the OPTIMADE API
-      "https://optimade.herokuapp.com": {  
+      "https://optimade.fly.dev": {
         // The OPTIMADE API response as if called with an infinite `page_limit`
-        "data": [  
+        "data": [
           {
             "id": "mpf_1",
             "type": "structures",
@@ -147,7 +147,7 @@ This is the same format as the cached results of the Python client:
 
 ```python
 from optimade.client import OptimadeClient
-client = OptimadeClient(base_urls="https://optimade.herokuapp.com")
+client = OptimadeClient(base_urls="https://optimade.fly.dev")
 client.get('nsites = 1')
 client.get('nsites = 2')
 print(client.all_results)
@@ -158,10 +158,10 @@ will return a dictionary with top-level keys:
 {
     "structures": {
         "nsites = 1": {
-            "https://optimade.herokuapp.com": {...}
+            "https://optimade.fly.dev": {...}
         },
         "nsites = 2": {
-            "https://optimade.herokuapp.com": {...}
+            "https://optimade.fly.dev": {...}
         }
     }
 }

optimade/server/config.py

@@ -153,7 +153,7 @@ class ServerConfig(BaseSettings):
         description="Which database backend to use out of the supported backends.",
     )
 
-    elastic_hosts: Optional[List[Dict]] = Field(
+    elastic_hosts: Optional[Union[str, List[str], Dict, List[Dict]]] = Field(
         None, description="Host settings to pass through to the `Elasticsearch` class."
     )
 

optimade/server/entry_collections/elasticsearch.py

@@ -72,9 +72,9 @@ def create_optimade_index(self) -> None:
             ]["properties"].pop(field)
         properties["id"] = {"type": "keyword"}
         body["mappings"]["properties"] = properties
-        self.client.indices.create(index=self.name, body=body, ignore=400)
+        self.client.indices.create(index=self.name, ignore=400, **body)
 
-        LOGGER.debug(f"Created Elastic index for {self.name!r} with body {body}")
+        LOGGER.debug(f"Created Elastic index for {self.name!r} with parameters {body}")
 
     @property
     def predefined_index(self) -> Dict[str, Any]:
@@ -94,7 +94,8 @@ def create_elastic_index_from_mapper(
             fields: The list of fields to use in the index.
 
         Returns:
-            The `body` parameter to pass to `client.indices.create(..., body=...)`.
+            The parameters to pass to `client.indices.create(...)` (previously
+                the 'body' parameters).
 
         """
         properties = {
@@ -135,15 +136,14 @@ def get_id(item):
 
         bulk(
             self.client,
-            [
+            (
                 {
                     "_index": self.name,
                     "_id": get_id(item),
-                    "_type": "_doc",
                     "_source": item,
                 }
                 for item in data
-            ],
+            ),
         )
 
     def _run_db_query(
@@ -177,9 +177,9 @@ def _run_db_query(
             for field, sort_dir in criteria.get("sort", {})
         ]
         if not elastic_sort:
-            elastic_sort = {
-                self.resource_mapper.get_backend_field("id"): {"order": "asc"}
-            }
+            elastic_sort = [
+                {self.resource_mapper.get_backend_field("id"): {"order": "asc"}}
+            ]
 
         search = search.sort(*elastic_sort)
 

requirements.txt

@@ -1,3 +1,4 @@
+elasticsearch==7.17.7
 elasticsearch-dsl==7.4.0
 email_validator==1.3.0
 fastapi==0.86.0

setup.cfg

@@ -3,3 +3,19 @@ ignore = E251,E252,E501,W503
 
 [flake8]
 ignore = E402,W503,E501,E203
+
+[tool:pytest]
+filterwarnings =
+    error
+    ignore:.*parameter is deprecated for the.*:DeprecationWarning
+    ignore:.*read_text is deprecated.*:DeprecationWarning
+    ignore:.*open_text is deprecated.*:DeprecationWarning
+    ignore:.*SixMetaPathImporter.*:ImportWarning
+    ignore:.*PY_SSIZE_T_CLEAN will be required for.*:DeprecationWarning
+    ignore:.*not found, cannot convert structure.*:
+    ignore:.*will default to setting mass to 1\.0\.:
+    ignore:.*is missing fields.*which are required if.*:
+    ignore:.*the imp module is deprecated in favour of importlib.*:DeprecationWarning
+    ignore:.*has an unrecognised prefix.*:
+testpaths = tests
+addopts = -rs

setup.py

@@ -18,7 +18,7 @@
 
 # Dependencies
 # Server minded
-elastic_deps = ["elasticsearch-dsl~=7.4,<8.0"]
+elastic_deps = ["elasticsearch-dsl~=7.4,<8.0", "elasticsearch~=7.17"]
 mongo_deps = ["pymongo>=3.12.1,<5", "mongomock~=4.1"]
 server_deps = [
     "uvicorn~=0.19",
@@ -101,7 +101,7 @@
     python_requires=">=3.8",
     install_requires=[
         "lark~=1.1",
-        "fastapi~=0.86",
+        "fastapi~=0.86.0",
         "pydantic~=1.10,>=1.10.2",
         "email_validator~=1.2",
         "requests~=2.28",

tests/adapters/structures/test_structures.py

@@ -1,15 +1,14 @@
 """Test Structure adapter"""
 import pytest
-
 from optimade.adapters import Structure
 from optimade.models import StructureResource
 
 try:
     import aiida  # noqa: F401
     import ase  # noqa: F401
+    import jarvis  # noqa: F401
     import numpy  # noqa: F401
     import pymatgen  # noqa: F401
-    import jarvis  # noqa: F401
 except ImportError:
     all_modules_found = False
 else:
@@ -34,6 +33,7 @@ def test_convert(structure):
     """Test convert() works
     Choose currently known entry type - must be updated if no longer available.
     """
+    pytest.importorskip("numpy")
     if not structure._type_converters:
         pytest.fail("_type_converters is seemingly empty. This should not be.")
 
@@ -102,6 +102,7 @@ def test_getattr_order(structure):
 def test_no_module_conversion(structure):
     """Make sure a warnings is raised and None is returned for conversions with non-existing modules"""
     import importlib
+
     from optimade.adapters.warnings import AdapterPackageNotFound
 
     CONVERSION_MAPPING = {