End to end runner.

.github/workflows/e2e.yml

@@ -0,0 +1,69 @@
+
+name: E2E
+
+on:
+  schedule:
+    # Runs at 02:57am every day.
+    - cron: '57 2 * * *'
+jobs:
+  # We only run the end to end tests if there have been new commits in the last
+  # 24 hours.
+  # https://stackoverflow.com/questions/63014786/how-to-schedule-a-github-actions-nightly-build-but-run-it-only-when-there-where
+  check_date:
+   runs-on: ubuntu-latest
+   name: Check latest commit
+   outputs:
+     should_run: ${{ steps.should_run.outputs.should_run }}
+   steps:
+     - uses: actions/checkout@v3
+
+     - name: print latest_commit
+       run: echo ${{ github.sha }}
+
+     - id: should_run
+       continue-on-error: true
+       name: Check latest commit is less than a day old.
+       if: ${{ github.event_name == 'schedule' }}
+       run: |
+          sha=$(git rev-list --after="24 hours" ${{ github.sha }})
+          if test -z $sha
+          then
+            echo "should_run=false" >> $GITHUB_OUTPUT
+          else
+            echo "should_run=true" >> $GITHUB_OUTPUT
+          fi
+
+  test:
+    needs: check_date
+    if: ${{ needs.check_date.outputs.should_run != 'false' }}
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        python-version: ["3.10"]
+    name: End to end tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Update environment
+        run: |
+          poetry env use ${{ matrix.python-version }} && poetry install --with=test,cloud,cloud_dev
+
+      - name: Run tests
+        run: |
+          poetry run modal token set \
+            --token-id ${{ secrets.MODAL_TOKEN_ID }} \
+            --token-secret ${{ secrets.MODAL_TOKEN_SECRET }}
+            # --env=confirm_solutions
+          poetry run python confirm/cloud/e2e_runner.py

confirm/cloud/clickhouse.py

@@ -390,9 +390,7 @@ def connect(
         """
         if job_id is None:
             config = get_ch_config(host, port, username, password)
-            test_host = keyring.get_password(
-                "clickhouse-confirm-test-host", os.environ["USER"]
-            )
+            test_host = get_ch_test_host()
             if not (
                 (test_host is not None and test_host in config["host"])
                 or "localhost" in config["host"]
@@ -440,9 +438,7 @@ def get_ch_config(host=None, port=None, username=None, password=None, database=N
         if "CLICKHOUSE_HOST" in os.environ:
             host = os.environ["CLICKHOUSE_HOST"]
         else:
-            host = keyring.get_password(
-                "clickhouse-confirm-test-host", os.environ["USER"]
-            )
+            host = get_ch_test_host()
     if port is None:
         if "CLICKHOUSE_PORT" in os.environ:
             port = os.environ["CLICKHOUSE_PORT"]
@@ -466,6 +462,13 @@ def get_ch_config(host=None, port=None, username=None, password=None, database=N
     )
 
 
+def get_ch_test_host():
+    if "CLICKHOUSE_TEST_HOST" in os.environ:
+        return os.environ["CLICKHOUSE_TEST_HOST"]
+    else:
+        return keyring.get_password("clickhouse-confirm-test-host", os.environ["USER"])
+
+
 def find_unique_job_id(client, attempts=3):
     for i in range(attempts):
         job_id = uuid.uuid4().hex
@@ -499,7 +502,7 @@ def clear_dbs(client):
     Args:
         client: _description_
     """
-    test_host = keyring.get_password("clickhouse-confirm-test-host", os.environ["USER"])
+    test_host = get_ch_test_host()
     if not (
         (test_host is not None and test_host in client.url) or "localhost" in client.url
     ):

confirm/cloud/e2e_runner.py

@@ -0,0 +1,49 @@
+import sys
+
+import modal
+import pytest
+
+import confirm.cloud.modal_util as modal_util
+
+stub = modal.Stub("e2e_runner")
+
+img = modal_util.get_image(dependency_groups=["test", "cloud"])
+
+
+def run_tests(argv=None):
+    if argv is None:
+        argv = ["--runslow", "tests", "imprint/tests"]
+    print(argv)
+    exitcode = pytest.main(argv)
+    print(exitcode)
+    return exitcode.value
+
+
+@stub.function(
+    image=img,
+    gpu=modal.gpu.A100(),
+    retries=0,
+    mounts=(
+        modal.create_package_mounts(["confirm", "imprint"])
+        + [
+            modal.Mount(local_dir="./tests", remote_dir="/root/tests"),
+            modal.Mount(local_dir="./imprint/tests", remote_dir="/root/imprint/tests"),
+            modal.Mount(
+                local_dir="./imprint/tutorials", remote_dir="/root/imprint/tutorials"
+            ),
+            modal.Mount(local_dir="./", remote_dir="/root", recursive=False),
+        ]
+    ),
+    timeout=60 * 60 * 1,
+    secrets=[modal.Secret.from_name("confirm-secrets")],
+)
+def run_cloud_tests(argv=None):
+    return run_tests(argv=argv)
+
+
+if __name__ == "__main__":
+    # run_tests()
+    with stub.run():
+        argv = None if len(sys.argv) == 1 else sys.argv[1:]
+        exitcode = run_cloud_tests.call(argv)
+    sys.exit(exitcode)

confirm/cloud/modal_secrets.py

@@ -0,0 +1,18 @@
+import os
+
+import keyring
+import modal
+
+
+secrets = dict()
+for (keyring_name, env_name) in [
+    ("clickhouse-confirm-test-host", "CLICKHOUSE_HOST"),
+    ("clickhouse-confirm-test-host", "CLICKHOUSE_TEST_HOST"),
+    ("clickhouse-confirm-test-password", "CLICKHOUSE_PASSWORD"),
+    ("upstash-confirm-coordinator-host", "REDIS_HOST"),
+    ("upstash-confirm-coordinator-password", "REDIS_PASSWORD"),
+]:
+    secrets[env_name] = keyring.get_password(keyring_name, os.environ["USER"])
+
+stub = modal.Stub("confirm-secrets")
+stub["secret"] = modal.Secret(secrets)

confirm/cloud/modal_util.py

@@ -3,7 +3,7 @@
 import modal
 
 
-def get_image():
+def get_image(dependency_groups=["cloud"]):
     poetry_dir = Path(__file__).resolve().parent.parent.parent
 
     context_files = {
@@ -17,7 +17,7 @@ def get_image():
         "COPY /.pyproject.toml /tmp/poetry/pyproject.toml",
         "RUN cd /tmp/poetry && \\ ",
         "  poetry config virtualenvs.create false && \\ ",
-        "  poetry install --with=cloud --no-root",
+        f"  poetry install --with={','.join(dependency_groups)} --no-root",
     ]
 
     return modal.Image.from_dockerhub(

docs/ci.md

@@ -1,6 +1,10 @@
 # Continuous Integration
 
-Related: [docs/packaging.md](./packaging.mdci.md)
+Related: [docs/packaging.md](./packaging.md)
+
+## Nightly end to end tests
+
+Our end to end tests are run nightly via the [.github/workflows/e2e.yml](../.github/workflows/e2e.yml) actions workflow which launches [confirm/cloud/e2e_runner.py](../confirm/cloud/e2e_runner.py). That script runs all tests including slow tests on a Modal server with a single GPU.
 
 ## Tools
 - renovatebot is used to maintain dependency version:
@@ -11,7 +15,7 @@ Related: [docs/packaging.md](./packaging.mdci.md)
 
 ## Clearing the Github Actions cache
 
-We cache lots of stuff. If you need to clear the cache, run this command. 
+We cache lots of stuff including our Python environments. If you need to clear the cache, run this command. 
 
 ```
 gh actions-cache list --limit 100 | tail -n +5 | awk '{print $1}' | tr '\n' '\0' | xargs -0 -n1 gh actions-cache delete --confirm

docs/packaging.md

@@ -48,6 +48,12 @@ However, without conda, we would:
 2. Lose access to the large repository of easy-to-install packages that are often not-at-all-easy-to-install with pip/poetry. (e.g. sage!) This is not important for production jobs but it's nice to have this access for development.
 3. Need to spend time updating various parts of our cloud setup, mainly the Dockerfiles. Updating Dockerfiles is painful because the iteration time is so slow.
 
+## How to update pyproject.toml package versions?
+
+- [The poetry plugin "poetry up" helps here.](https://github.com/MousaZeidBaker/poetry-plugin-up)
+- `poetry up --with=test,cloud,dev,cloud_dev`
+- [Lots of discussion in this issue](https://github.com/python-poetry/poetry/issues/461)
+
 ## Notes
 
 - [JAX doesn't have a "normal" package registry and doesn't use PyPI](https://github.com/google/jax/issues/5410)

imprint/tutorials/ztest.md

@@ -34,6 +34,8 @@ plt.show()
 ```
 
 ```python
+true_drug_effect = 0
+K = 10000
 samples = scipy.stats.norm.rvs(loc=true_drug_effect, size=K)
 ```