Separating the database and caching infrastructure. (#180)

* Starting on caching.

* Successfully continuing an adagrid run from a database.

* Caching configuration and null hypotheses in duckdb.

* Use pottery/redis/upstash for locks and nextid.

* Model data caching (e.g. rng).

* birthtime, worker_id for tiles

* ClickhouseStore

* Why are the ids inconsistent?

* Why are the ids inconsistent?

* Why are the ids inconsistent?

* Adding some documentation to Clickhouse and Store.

* Add keyring dependency.

* Mock keyring when no keyring is available.

* Allow get_password to return None.

.github/workflows/test.yml

@@ -51,14 +51,14 @@ jobs:
 
       - name: Update environment
         run: |
-          poetry env use ${{ matrix.python-version }} && poetry install --with=test
+          poetry env use ${{ matrix.python-version }} && poetry install --with=test,cloud
           
       - name: Get changed files
         id: changed-files
         uses: tj-actions/[email protected]
         if: github.ref != 'refs/heads/main'
 
-      - name: Run all pre-commit checks on change files.
+      - name: Run all pre-commit checks on changed files.
         run: poetry run pre-commit run --color=always --files ${{ steps.changed-files.outputs.all_changed_files }}
         if: github.ref != 'refs/heads/main'
 
@@ -67,7 +67,7 @@ jobs:
         if: github.ref == 'refs/heads/main'
 
       - name: Pytest
-        run: poetry run pytest -n auto 
+        run: poetry run pytest -n auto -s
 
       # TODO: this doesn't currently work because only the imprint repo has the SLACK_BOT_TOKEN
       # - name: Report failures on Slack

.pre-commit-config.yaml

@@ -1,11 +1,20 @@
 fail_fast: false
 repos:
+  # We have black here twice because the first black block runs black on
+  # Jupyter notebooks. And we don't want to run black on notebooks in the
+  # research folder because 1) it's slow. 2) it can overwrite a notebook in
+  # current use causing the session to be lost. 3) there's no reason, those
+  # notebooks are exploration anyway.
   - repo: https://github.com/psf/black
     rev: 22.10.0
     hooks:
       - id: black-jupyter
         language_version: python3
         exclude: research/
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
   - repo: https://github.com/pycqa/flake8
     rev: 5.0.4
     hooks:

.vscode/settings.json

@@ -3,6 +3,7 @@
   "python.testing.pytestEnabled": true,
   "python.linting.pylintEnabled": false,
   "python.linting.flake8Enabled": true,
+  "python.linting.flake8Args": ["--ignore=E1,E2,E3,E4,E5,W1,W2,W3,W4,W5"],
   "python.linting.enabled": true,
   "python.formatting.provider": "black",
   "autoDocstring.docstringFormat": "google-notypes",